2024-09-17T23:31:40.007 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-17T23:31:40.030 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T23:31:40.127 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601 branch: squid description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid supported-random-distro$/{ubuntu_latest} workloads/compare-mirror-images-nbd} email: ceph-qa@ceph.com first_in_suite: false job_id: '7909601' kernel: kdb: 1 sha1: distro ktype: distro last_in_suite: false machine_type: smithi meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each name: teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 3 size: 30 os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: squid ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.4: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.5: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.6: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluefs allocator: hybrid bluestore allocator: hybrid bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: extra_packages: - rbd-nbd flavor: default sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f extra_system_packages: - pv thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: squid sha1: b0d356b98e08f6bc59ef15652f0d0e4298fc83c7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 1983 sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f sleep_before_teardown: 0 subset: 79/128 suite: rbd suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_b0d356b98e08f6bc59ef15652f0d0e4298fc83c7/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: b0d356b98e08f6bc59ef15652f0d0e4298fc83c7 targets: smithi028.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJ6R7D/dpH5Q/kJUJlgC8va/DNhJaP+PmCcJJPaHFjdILYt4sac7Kdqus1jDVzYR5Dp/oPvfM3MhfFS26p5pLlo= smithi049.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKHINc9zC/b1hH/hNMeqnZWqar4lztbQ1kZBO25TL4fCb59Evgo2aYxq63Ywrhh+do4yrTzxrlD+89bwhBYec8o= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - workunit: clients: cluster1.client.mirror: - rbd/compare_mirror_images.sh env: RBD_DEVICE_TYPE: nbd timeout: 3h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-17_21:16:02 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-17T23:31:40.127 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_b0d356b98e08f6bc59ef15652f0d0e4298fc83c7/qa; will attempt to use it 2024-09-17T23:31:40.128 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_b0d356b98e08f6bc59ef15652f0d0e4298fc83c7/qa/tasks 2024-09-17T23:31:40.128 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-17T23:31:40.130 INFO:teuthology.task.internal:Checking packages... 2024-09-17T23:31:40.151 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'c2ddf9721ef254645201cdbabd9c8f49c30a586f' 2024-09-17T23:31:40.151 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-17T23:31:40.151 INFO:teuthology.packaging:ref: None 2024-09-17T23:31:40.151 INFO:teuthology.packaging:tag: None 2024-09-17T23:31:40.151 INFO:teuthology.packaging:branch: squid 2024-09-17T23:31:40.151 INFO:teuthology.packaging:sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T23:31:40.152 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=squid 2024-09-17T23:31:40.355 INFO:teuthology.task.internal:Found packages for ceph version 19.2.0-1jammy 2024-09-17T23:31:40.357 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-17T23:31:40.363 INFO:teuthology.task.internal:no buildpackages task found 2024-09-17T23:31:40.364 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-17T23:31:40.383 INFO:teuthology.task.internal:Saving configuration 2024-09-17T23:31:40.400 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-17T23:31:40.407 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-17T23:31:40.435 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi028.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 23:24:32.072380', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB9wgBhoUGP5NKQEWmNsBGlHvlgi3jaD7FCOwSZyaYCQ'} 2024-09-17T23:31:40.458 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi049.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 23:24:32.073813', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIdOvAsm1mXvR3pe+W/7tfqoxSQ3miTW+8O1xOsnmabc'} 2024-09-17T23:31:40.458 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-17T23:31:40.464 INFO:teuthology.task.internal:roles: ubuntu@smithi028.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2024-09-17T23:31:40.464 INFO:teuthology.task.internal:roles: ubuntu@smithi049.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2024-09-17T23:31:40.464 INFO:teuthology.run_tasks:Running task console_log... 2024-09-17T23:31:40.520 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f9cffca7760>, signals=[15]) 2024-09-17T23:31:40.521 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-17T23:31:40.527 INFO:teuthology.task.internal:Opening connections... 2024-09-17T23:31:40.527 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi028.front.sepia.ceph.com 2024-09-17T23:31:40.528 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:31:40.607 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi049.front.sepia.ceph.com 2024-09-17T23:31:40.608 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi049.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:31:40.688 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-17T23:31:40.694 DEBUG:teuthology.orchestra.run.smithi028:> uname -m 2024-09-17T23:31:40.699 INFO:teuthology.orchestra.run.smithi028.stdout:x86_64 2024-09-17T23:31:40.700 DEBUG:teuthology.orchestra.run.smithi028:> cat /etc/os-release 2024-09-17T23:31:40.747 INFO:teuthology.orchestra.run.smithi028.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-09-17T23:31:40.747 INFO:teuthology.orchestra.run.smithi028.stdout:NAME="Ubuntu" 2024-09-17T23:31:40.747 INFO:teuthology.orchestra.run.smithi028.stdout:VERSION_ID="22.04" 2024-09-17T23:31:40.747 INFO:teuthology.orchestra.run.smithi028.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-09-17T23:31:40.747 INFO:teuthology.orchestra.run.smithi028.stdout:VERSION_CODENAME=jammy 2024-09-17T23:31:40.747 INFO:teuthology.orchestra.run.smithi028.stdout:ID=ubuntu 2024-09-17T23:31:40.747 INFO:teuthology.orchestra.run.smithi028.stdout:ID_LIKE=debian 2024-09-17T23:31:40.747 INFO:teuthology.orchestra.run.smithi028.stdout:HOME_URL="https://www.ubuntu.com/" 2024-09-17T23:31:40.747 INFO:teuthology.orchestra.run.smithi028.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-09-17T23:31:40.748 INFO:teuthology.orchestra.run.smithi028.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-09-17T23:31:40.748 INFO:teuthology.orchestra.run.smithi028.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-09-17T23:31:40.748 INFO:teuthology.orchestra.run.smithi028.stdout:UBUNTU_CODENAME=jammy 2024-09-17T23:31:40.748 INFO:teuthology.lock.ops:Updating smithi028.front.sepia.ceph.com on lock server 2024-09-17T23:31:40.773 DEBUG:teuthology.orchestra.run.smithi049:> uname -m 2024-09-17T23:31:40.778 INFO:teuthology.orchestra.run.smithi049.stdout:x86_64 2024-09-17T23:31:40.778 DEBUG:teuthology.orchestra.run.smithi049:> cat /etc/os-release 2024-09-17T23:31:40.823 INFO:teuthology.orchestra.run.smithi049.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-09-17T23:31:40.823 INFO:teuthology.orchestra.run.smithi049.stdout:NAME="Ubuntu" 2024-09-17T23:31:40.824 INFO:teuthology.orchestra.run.smithi049.stdout:VERSION_ID="22.04" 2024-09-17T23:31:40.824 INFO:teuthology.orchestra.run.smithi049.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-09-17T23:31:40.824 INFO:teuthology.orchestra.run.smithi049.stdout:VERSION_CODENAME=jammy 2024-09-17T23:31:40.824 INFO:teuthology.orchestra.run.smithi049.stdout:ID=ubuntu 2024-09-17T23:31:40.824 INFO:teuthology.orchestra.run.smithi049.stdout:ID_LIKE=debian 2024-09-17T23:31:40.824 INFO:teuthology.orchestra.run.smithi049.stdout:HOME_URL="https://www.ubuntu.com/" 2024-09-17T23:31:40.824 INFO:teuthology.orchestra.run.smithi049.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-09-17T23:31:40.824 INFO:teuthology.orchestra.run.smithi049.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-09-17T23:31:40.824 INFO:teuthology.orchestra.run.smithi049.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-09-17T23:31:40.824 INFO:teuthology.orchestra.run.smithi049.stdout:UBUNTU_CODENAME=jammy 2024-09-17T23:31:40.824 INFO:teuthology.lock.ops:Updating smithi049.front.sepia.ceph.com on lock server 2024-09-17T23:31:40.839 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-17T23:31:40.848 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-17T23:31:40.854 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-17T23:31:40.854 DEBUG:teuthology.orchestra.run.smithi028:> test '!' -e /home/ubuntu/cephtest 2024-09-17T23:31:40.857 DEBUG:teuthology.orchestra.run.smithi049:> test '!' -e /home/ubuntu/cephtest 2024-09-17T23:31:40.870 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-17T23:31:40.877 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-17T23:31:40.877 DEBUG:teuthology.orchestra.run.smithi028:> test -z $(ls -A /var/lib/ceph) 2024-09-17T23:31:40.904 DEBUG:teuthology.orchestra.run.smithi049:> test -z $(ls -A /var/lib/ceph) 2024-09-17T23:31:40.930 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-17T23:31:40.982 INFO:teuthology.run_tasks:Running task kernel... 2024-09-17T23:31:40.996 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-17T23:31:40.996 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'kdb': 1, 'sha1': 'distro'}, 'cluster2.mon.a': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-17T23:31:40.997 DEBUG:teuthology.orchestra.run.smithi028:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T23:31:40.997 DEBUG:teuthology.orchestra.run.smithi049:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T23:31:41.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T23:31:41.002 DEBUG:teuthology.orchestra.run.smithi028:> uname -r 2024-09-17T23:31:41.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T23:31:41.003 DEBUG:teuthology.orchestra.run.smithi049:> uname -r 2024-09-17T23:31:41.048 INFO:teuthology.orchestra.run.smithi049.stdout:5.15.0-56-generic 2024-09-17T23:31:41.048 INFO:teuthology.task.kernel:Running kernel on smithi049: 5.15.0-56-generic 2024-09-17T23:31:41.048 DEBUG:teuthology.orchestra.run.smithi049:> sudo apt-get clean 2024-09-17T23:31:41.051 INFO:teuthology.orchestra.run.smithi028.stdout:5.15.0-56-generic 2024-09-17T23:31:41.051 INFO:teuthology.task.kernel:Running kernel on smithi028: 5.15.0-56-generic 2024-09-17T23:31:41.051 DEBUG:teuthology.orchestra.run.smithi028:> sudo apt-get clean 2024-09-17T23:31:41.164 DEBUG:teuthology.orchestra.run.smithi028:> sudo apt-get update 2024-09-17T23:31:41.205 DEBUG:teuthology.orchestra.run.smithi049:> sudo apt-get update 2024-09-17T23:31:41.313 INFO:teuthology.orchestra.run.smithi028.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-17T23:31:41.314 INFO:teuthology.orchestra.run.smithi028.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-09-17T23:31:41.314 INFO:teuthology.orchestra.run.smithi028.stdout:Get:3 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-09-17T23:31:41.361 INFO:teuthology.orchestra.run.smithi049.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-17T23:31:41.379 INFO:teuthology.orchestra.run.smithi049.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-09-17T23:31:41.389 INFO:teuthology.orchestra.run.smithi028.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-09-17T23:31:41.443 INFO:teuthology.orchestra.run.smithi049.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-09-17T23:31:41.478 INFO:teuthology.orchestra.run.smithi049.stdout:Get:4 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-09-17T23:31:41.577 INFO:teuthology.orchestra.run.smithi028.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [544 kB] 2024-09-17T23:31:41.592 INFO:teuthology.orchestra.run.smithi049.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,058 kB] 2024-09-17T23:31:41.668 INFO:teuthology.orchestra.run.smithi028.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [701 kB] 2024-09-17T23:31:41.668 INFO:teuthology.orchestra.run.smithi028.stdout:Get:7 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,839 kB] 2024-09-17T23:31:41.689 INFO:teuthology.orchestra.run.smithi049.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [701 kB] 2024-09-17T23:31:41.696 INFO:teuthology.orchestra.run.smithi049.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [355 kB] 2024-09-17T23:31:41.699 INFO:teuthology.orchestra.run.smithi049.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.8 kB] 2024-09-17T23:31:41.699 INFO:teuthology.orchestra.run.smithi049.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [39.3 kB] 2024-09-17T23:31:41.699 INFO:teuthology.orchestra.run.smithi049.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,495 kB] 2024-09-17T23:31:41.706 INFO:teuthology.orchestra.run.smithi028.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [298 kB] 2024-09-17T23:31:41.709 INFO:teuthology.orchestra.run.smithi028.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.3 kB] 2024-09-17T23:31:41.709 INFO:teuthology.orchestra.run.smithi028.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,431 kB] 2024-09-17T23:31:41.716 INFO:teuthology.orchestra.run.smithi049.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [429 kB] 2024-09-17T23:31:41.719 INFO:teuthology.orchestra.run.smithi049.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [616 B] 2024-09-17T23:31:41.719 INFO:teuthology.orchestra.run.smithi049.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,124 kB] 2024-09-17T23:31:41.727 INFO:teuthology.orchestra.run.smithi049.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [732 kB] 2024-09-17T23:31:41.728 INFO:teuthology.orchestra.run.smithi028.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.7 kB] 2024-09-17T23:31:41.729 INFO:teuthology.orchestra.run.smithi028.stdout:Get:12 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [418 kB] 2024-09-17T23:31:41.730 INFO:teuthology.orchestra.run.smithi028.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,058 kB] 2024-09-17T23:31:41.732 INFO:teuthology.orchestra.run.smithi049.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [261 kB] 2024-09-17T23:31:41.732 INFO:teuthology.orchestra.run.smithi028.stdout:Get:14 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [584 B] 2024-09-17T23:31:41.732 INFO:teuthology.orchestra.run.smithi028.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [628 kB] 2024-09-17T23:31:41.734 INFO:teuthology.orchestra.run.smithi049.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [26.1 kB] 2024-09-17T23:31:41.734 INFO:teuthology.orchestra.run.smithi049.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-09-17T23:31:41.734 INFO:teuthology.orchestra.run.smithi049.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,752 B] 2024-09-17T23:31:41.735 INFO:teuthology.orchestra.run.smithi049.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-09-17T23:31:41.737 INFO:teuthology.orchestra.run.smithi028.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [903 kB] 2024-09-17T23:31:41.743 INFO:teuthology.orchestra.run.smithi028.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [177 kB] 2024-09-17T23:31:41.744 INFO:teuthology.orchestra.run.smithi028.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [19.3 kB] 2024-09-17T23:31:41.744 INFO:teuthology.orchestra.run.smithi028.stdout:Get:19 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-09-17T23:31:41.750 INFO:teuthology.orchestra.run.smithi049.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-09-17T23:31:41.768 INFO:teuthology.orchestra.run.smithi028.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [355 kB] 2024-09-17T23:31:41.768 INFO:teuthology.orchestra.run.smithi028.stdout:Get:21 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-09-17T23:31:41.768 INFO:teuthology.orchestra.run.smithi028.stdout:Get:22 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-09-17T23:31:41.769 INFO:teuthology.orchestra.run.smithi028.stdout:Get:23 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-09-17T23:31:41.772 INFO:teuthology.orchestra.run.smithi028.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.8 kB] 2024-09-17T23:31:41.773 INFO:teuthology.orchestra.run.smithi028.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,495 kB] 2024-09-17T23:31:41.775 INFO:teuthology.orchestra.run.smithi049.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.9 kB] 2024-09-17T23:31:41.776 INFO:teuthology.orchestra.run.smithi049.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.8 kB] 2024-09-17T23:31:41.776 INFO:teuthology.orchestra.run.smithi049.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.1 kB] 2024-09-17T23:31:41.777 INFO:teuthology.orchestra.run.smithi049.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-09-17T23:31:41.777 INFO:teuthology.orchestra.run.smithi049.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.8 kB] 2024-09-17T23:31:41.778 INFO:teuthology.orchestra.run.smithi049.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.3 kB] 2024-09-17T23:31:41.778 INFO:teuthology.orchestra.run.smithi049.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-09-17T23:31:41.778 INFO:teuthology.orchestra.run.smithi049.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-09-17T23:31:41.802 INFO:teuthology.orchestra.run.smithi028.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [39.3 kB] 2024-09-17T23:31:41.803 INFO:teuthology.orchestra.run.smithi028.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [429 kB] 2024-09-17T23:31:41.808 INFO:teuthology.orchestra.run.smithi028.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [616 B] 2024-09-17T23:31:41.808 INFO:teuthology.orchestra.run.smithi028.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [732 kB] 2024-09-17T23:31:41.812 INFO:teuthology.orchestra.run.smithi028.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,124 kB] 2024-09-17T23:31:41.819 INFO:teuthology.orchestra.run.smithi028.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [261 kB] 2024-09-17T23:31:41.821 INFO:teuthology.orchestra.run.smithi028.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [26.1 kB] 2024-09-17T23:31:41.821 INFO:teuthology.orchestra.run.smithi028.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-09-17T23:31:41.821 INFO:teuthology.orchestra.run.smithi028.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,752 B] 2024-09-17T23:31:41.823 INFO:teuthology.orchestra.run.smithi028.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-09-17T23:31:41.824 INFO:teuthology.orchestra.run.smithi028.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-09-17T23:31:41.839 INFO:teuthology.orchestra.run.smithi028.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.8 kB] 2024-09-17T23:31:41.839 INFO:teuthology.orchestra.run.smithi028.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.9 kB] 2024-09-17T23:31:41.840 INFO:teuthology.orchestra.run.smithi028.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.1 kB] 2024-09-17T23:31:41.840 INFO:teuthology.orchestra.run.smithi028.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-09-17T23:31:41.840 INFO:teuthology.orchestra.run.smithi028.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.8 kB] 2024-09-17T23:31:41.841 INFO:teuthology.orchestra.run.smithi028.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.3 kB] 2024-09-17T23:31:41.841 INFO:teuthology.orchestra.run.smithi028.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-09-17T23:31:41.841 INFO:teuthology.orchestra.run.smithi028.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-09-17T23:31:41.925 INFO:teuthology.orchestra.run.smithi049.stdout:Get:29 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,839 kB] 2024-09-17T23:31:42.349 INFO:teuthology.orchestra.run.smithi049.stdout:Get:30 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [544 kB] 2024-09-17T23:31:42.365 INFO:teuthology.orchestra.run.smithi049.stdout:Get:31 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [298 kB] 2024-09-17T23:31:42.374 INFO:teuthology.orchestra.run.smithi049.stdout:Get:32 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.3 kB] 2024-09-17T23:31:42.374 INFO:teuthology.orchestra.run.smithi049.stdout:Get:33 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,431 kB] 2024-09-17T23:31:42.471 INFO:teuthology.orchestra.run.smithi049.stdout:Get:34 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.7 kB] 2024-09-17T23:31:42.472 INFO:teuthology.orchestra.run.smithi049.stdout:Get:35 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [418 kB] 2024-09-17T23:31:42.483 INFO:teuthology.orchestra.run.smithi049.stdout:Get:36 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [584 B] 2024-09-17T23:31:42.483 INFO:teuthology.orchestra.run.smithi049.stdout:Get:37 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [628 kB] 2024-09-17T23:31:42.499 INFO:teuthology.orchestra.run.smithi049.stdout:Get:38 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [903 kB] 2024-09-17T23:31:42.528 INFO:teuthology.orchestra.run.smithi049.stdout:Get:39 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [177 kB] 2024-09-17T23:31:42.534 INFO:teuthology.orchestra.run.smithi049.stdout:Get:40 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [19.3 kB] 2024-09-17T23:31:42.535 INFO:teuthology.orchestra.run.smithi049.stdout:Get:41 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-09-17T23:31:42.536 INFO:teuthology.orchestra.run.smithi049.stdout:Get:42 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-09-17T23:31:42.615 INFO:teuthology.orchestra.run.smithi049.stdout:Get:43 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-09-17T23:31:42.615 INFO:teuthology.orchestra.run.smithi049.stdout:Get:44 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-09-17T23:31:48.320 INFO:teuthology.orchestra.run.smithi049.stdout:Fetched 16.2 MB in 2s (7,148 kB/s) 2024-09-17T23:31:48.411 INFO:teuthology.orchestra.run.smithi028.stdout:Fetched 16.2 MB in 2s (7,058 kB/s) 2024-09-17T23:31:49.775 INFO:teuthology.orchestra.run.smithi028.stdout:Reading package lists... 2024-09-17T23:31:49.794 DEBUG:teuthology.orchestra.run.smithi028:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-17T23:31:49.823 INFO:teuthology.orchestra.run.smithi049.stdout:Reading package lists... 2024-09-17T23:31:49.843 DEBUG:teuthology.orchestra.run.smithi049:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-17T23:31:49.858 INFO:teuthology.orchestra.run.smithi028.stdout:Reading package lists... 2024-09-17T23:31:49.907 INFO:teuthology.orchestra.run.smithi049.stdout:Reading package lists... 2024-09-17T23:31:50.077 INFO:teuthology.orchestra.run.smithi028.stdout:Building dependency tree... 2024-09-17T23:31:50.078 INFO:teuthology.orchestra.run.smithi028.stdout:Reading state information... 2024-09-17T23:31:50.130 INFO:teuthology.orchestra.run.smithi049.stdout:Building dependency tree... 2024-09-17T23:31:50.131 INFO:teuthology.orchestra.run.smithi049.stdout:Reading state information... 2024-09-17T23:31:50.274 INFO:teuthology.orchestra.run.smithi028.stdout:The following packages were automatically installed and are no longer required: 2024-09-17T23:31:50.274 INFO:teuthology.orchestra.run.smithi028.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-17T23:31:50.274 INFO:teuthology.orchestra.run.smithi028.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-17T23:31:50.275 INFO:teuthology.orchestra.run.smithi028.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-17T23:31:50.275 INFO:teuthology.orchestra.run.smithi028.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-09-17T23:31:50.275 INFO:teuthology.orchestra.run.smithi028.stdout: python2.7-minimal 2024-09-17T23:31:50.275 INFO:teuthology.orchestra.run.smithi028.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-17T23:31:50.277 INFO:teuthology.orchestra.run.smithi028.stdout:The following additional packages will be installed: 2024-09-17T23:31:50.277 INFO:teuthology.orchestra.run.smithi028.stdout: linux-generic linux-headers-5.15.0-122 linux-headers-5.15.0-122-generic 2024-09-17T23:31:50.277 INFO:teuthology.orchestra.run.smithi028.stdout: linux-headers-generic linux-image-5.15.0-122-generic 2024-09-17T23:31:50.277 INFO:teuthology.orchestra.run.smithi028.stdout: linux-modules-5.15.0-122-generic linux-modules-extra-5.15.0-122-generic 2024-09-17T23:31:50.279 INFO:teuthology.orchestra.run.smithi028.stdout:Suggested packages: 2024-09-17T23:31:50.279 INFO:teuthology.orchestra.run.smithi028.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-09-17T23:31:50.279 INFO:teuthology.orchestra.run.smithi028.stdout:Recommended packages: 2024-09-17T23:31:50.279 INFO:teuthology.orchestra.run.smithi028.stdout: thermald 2024-09-17T23:31:50.330 INFO:teuthology.orchestra.run.smithi028.stdout:The following NEW packages will be installed: 2024-09-17T23:31:50.330 INFO:teuthology.orchestra.run.smithi028.stdout: linux-headers-5.15.0-122 linux-headers-5.15.0-122-generic 2024-09-17T23:31:50.330 INFO:teuthology.orchestra.run.smithi028.stdout: linux-image-5.15.0-122-generic linux-modules-5.15.0-122-generic 2024-09-17T23:31:50.330 INFO:teuthology.orchestra.run.smithi028.stdout: linux-modules-extra-5.15.0-122-generic 2024-09-17T23:31:50.332 INFO:teuthology.orchestra.run.smithi028.stdout:The following packages will be upgraded: 2024-09-17T23:31:50.332 INFO:teuthology.orchestra.run.smithi028.stdout: linux-generic linux-headers-generic linux-image-generic 2024-09-17T23:31:50.333 INFO:teuthology.orchestra.run.smithi049.stdout:The following packages were automatically installed and are no longer required: 2024-09-17T23:31:50.333 INFO:teuthology.orchestra.run.smithi049.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-17T23:31:50.334 INFO:teuthology.orchestra.run.smithi049.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-17T23:31:50.334 INFO:teuthology.orchestra.run.smithi049.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-17T23:31:50.334 INFO:teuthology.orchestra.run.smithi049.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-09-17T23:31:50.335 INFO:teuthology.orchestra.run.smithi049.stdout: python2.7-minimal 2024-09-17T23:31:50.335 INFO:teuthology.orchestra.run.smithi049.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-17T23:31:50.337 INFO:teuthology.orchestra.run.smithi049.stdout:The following additional packages will be installed: 2024-09-17T23:31:50.337 INFO:teuthology.orchestra.run.smithi049.stdout: linux-generic linux-headers-5.15.0-122 linux-headers-5.15.0-122-generic 2024-09-17T23:31:50.337 INFO:teuthology.orchestra.run.smithi049.stdout: linux-headers-generic linux-image-5.15.0-122-generic 2024-09-17T23:31:50.337 INFO:teuthology.orchestra.run.smithi049.stdout: linux-modules-5.15.0-122-generic linux-modules-extra-5.15.0-122-generic 2024-09-17T23:31:50.339 INFO:teuthology.orchestra.run.smithi049.stdout:Suggested packages: 2024-09-17T23:31:50.339 INFO:teuthology.orchestra.run.smithi049.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-09-17T23:31:50.339 INFO:teuthology.orchestra.run.smithi049.stdout:Recommended packages: 2024-09-17T23:31:50.339 INFO:teuthology.orchestra.run.smithi049.stdout: thermald 2024-09-17T23:31:50.393 INFO:teuthology.orchestra.run.smithi049.stdout:The following NEW packages will be installed: 2024-09-17T23:31:50.394 INFO:teuthology.orchestra.run.smithi049.stdout: linux-headers-5.15.0-122 linux-headers-5.15.0-122-generic 2024-09-17T23:31:50.394 INFO:teuthology.orchestra.run.smithi049.stdout: linux-image-5.15.0-122-generic linux-modules-5.15.0-122-generic 2024-09-17T23:31:50.394 INFO:teuthology.orchestra.run.smithi049.stdout: linux-modules-extra-5.15.0-122-generic 2024-09-17T23:31:50.396 INFO:teuthology.orchestra.run.smithi049.stdout:The following packages will be upgraded: 2024-09-17T23:31:50.396 INFO:teuthology.orchestra.run.smithi049.stdout: linux-generic linux-headers-generic linux-image-generic 2024-09-17T23:31:50.449 INFO:teuthology.orchestra.run.smithi049.stdout:3 upgraded, 5 newly installed, 0 to remove and 344 not upgraded. 2024-09-17T23:31:50.449 INFO:teuthology.orchestra.run.smithi049.stdout:Need to get 113 MB of archives. 2024-09-17T23:31:50.449 INFO:teuthology.orchestra.run.smithi049.stdout:After this operation, 584 MB of additional disk space will be used. 2024-09-17T23:31:50.449 INFO:teuthology.orchestra.run.smithi049.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-122-generic amd64 5.15.0-122.132 [22.7 MB] 2024-09-17T23:31:50.514 INFO:teuthology.orchestra.run.smithi028.stdout:3 upgraded, 5 newly installed, 0 to remove and 344 not upgraded. 2024-09-17T23:31:50.514 INFO:teuthology.orchestra.run.smithi028.stdout:Need to get 113 MB of archives. 2024-09-17T23:31:50.514 INFO:teuthology.orchestra.run.smithi028.stdout:After this operation, 584 MB of additional disk space will be used. 2024-09-17T23:31:50.514 INFO:teuthology.orchestra.run.smithi028.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-122-generic amd64 5.15.0-122.132 [22.7 MB] 2024-09-17T23:31:50.906 INFO:teuthology.orchestra.run.smithi049.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-122-generic amd64 5.15.0-122.132 [11.5 MB] 2024-09-17T23:31:51.027 INFO:teuthology.orchestra.run.smithi049.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-122-generic amd64 5.15.0-122.132 [63.9 MB] 2024-09-17T23:31:51.820 INFO:teuthology.orchestra.run.smithi049.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.122.122 [1,692 B] 2024-09-17T23:31:51.820 INFO:teuthology.orchestra.run.smithi049.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.122.122 [2,436 B] 2024-09-17T23:31:51.820 INFO:teuthology.orchestra.run.smithi049.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-122 all 5.15.0-122.132 [12.3 MB] 2024-09-17T23:31:51.941 INFO:teuthology.orchestra.run.smithi028.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-122-generic amd64 5.15.0-122.132 [11.5 MB] 2024-09-17T23:31:51.948 INFO:teuthology.orchestra.run.smithi049.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-122-generic amd64 5.15.0-122.132 [2,827 kB] 2024-09-17T23:31:51.969 INFO:teuthology.orchestra.run.smithi049.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.122.122 [2,294 B] 2024-09-17T23:31:52.298 INFO:teuthology.orchestra.run.smithi028.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-122-generic amd64 5.15.0-122.132 [63.9 MB] 2024-09-17T23:31:52.380 INFO:teuthology.orchestra.run.smithi049.stdout:Fetched 113 MB in 2s (72.4 MB/s) 2024-09-17T23:31:52.568 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package linux-modules-5.15.0-122-generic. 2024-09-17T23:31:54.535 INFO:teuthology.orchestra.run.smithi028.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.122.122 [1,692 B] 2024-09-17T23:31:54.536 INFO:teuthology.orchestra.run.smithi028.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.122.122 [2,436 B] 2024-09-17T23:31:54.536 INFO:teuthology.orchestra.run.smithi028.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-122 all 5.15.0-122.132 [12.3 MB] 2024-09-17T23:31:54.541 INFO:teuthology.orchestra.run.smithi049.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-09-17T23:31:54.545 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../0-linux-modules-5.15.0-122-generic_5.15.0-122.132_amd64.deb ... 2024-09-17T23:31:54.607 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking linux-modules-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:31:54.904 INFO:teuthology.orchestra.run.smithi028.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-122-generic amd64 5.15.0-122.132 [2,827 kB] 2024-09-17T23:31:54.988 INFO:teuthology.orchestra.run.smithi028.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.122.122 [2,294 B] 2024-09-17T23:31:55.406 INFO:teuthology.orchestra.run.smithi028.stdout:Fetched 113 MB in 5s (24.4 MB/s) 2024-09-17T23:31:55.612 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package linux-modules-5.15.0-122-generic. 2024-09-17T23:31:56.867 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package linux-image-5.15.0-122-generic. 2024-09-17T23:31:56.888 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../1-linux-image-5.15.0-122-generic_5.15.0-122.132_amd64.deb ... 2024-09-17T23:31:57.233 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking linux-image-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:31:57.589 INFO:teuthology.orchestra.run.smithi028.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-09-17T23:31:57.593 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../0-linux-modules-5.15.0-122-generic_5.15.0-122.132_amd64.deb ... 2024-09-17T23:31:57.676 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking linux-modules-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:31:57.712 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-122-generic. 2024-09-17T23:31:57.733 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-122-generic_5.15.0-122.132_amd64.deb ... 2024-09-17T23:31:57.770 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking linux-modules-extra-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:31:59.969 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package linux-image-5.15.0-122-generic. 2024-09-17T23:31:59.991 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../1-linux-image-5.15.0-122-generic_5.15.0-122.132_amd64.deb ... 2024-09-17T23:32:00.087 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking linux-image-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:00.556 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-122-generic. 2024-09-17T23:32:00.576 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-122-generic_5.15.0-122.132_amd64.deb ... 2024-09-17T23:32:00.614 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking linux-modules-extra-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:04.232 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../3-linux-generic_5.15.0.122.122_amd64.deb ... 2024-09-17T23:32:04.358 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking linux-generic (5.15.0.122.122) over (5.15.0.56.54) ... 2024-09-17T23:32:04.691 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.122.122_amd64.deb ... 2024-09-17T23:32:04.811 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking linux-image-generic (5.15.0.122.122) over (5.15.0.56.54) ... 2024-09-17T23:32:05.081 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package linux-headers-5.15.0-122. 2024-09-17T23:32:05.103 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../5-linux-headers-5.15.0-122_5.15.0-122.132_all.deb ... 2024-09-17T23:32:05.139 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking linux-headers-5.15.0-122 (5.15.0-122.132) ... 2024-09-17T23:32:06.948 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../3-linux-generic_5.15.0.122.122_amd64.deb ... 2024-09-17T23:32:07.049 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking linux-generic (5.15.0.122.122) over (5.15.0.56.54) ... 2024-09-17T23:32:07.357 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.122.122_amd64.deb ... 2024-09-17T23:32:07.502 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking linux-image-generic (5.15.0.122.122) over (5.15.0.56.54) ... 2024-09-17T23:32:07.805 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package linux-headers-5.15.0-122. 2024-09-17T23:32:07.816 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../5-linux-headers-5.15.0-122_5.15.0-122.132_all.deb ... 2024-09-17T23:32:07.872 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking linux-headers-5.15.0-122 (5.15.0-122.132) ... 2024-09-17T23:32:10.461 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package linux-headers-5.15.0-122-generic. 2024-09-17T23:32:10.498 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../6-linux-headers-5.15.0-122-generic_5.15.0-122.132_amd64.deb ... 2024-09-17T23:32:10.536 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking linux-headers-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:13.553 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package linux-headers-5.15.0-122-generic. 2024-09-17T23:32:13.589 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../6-linux-headers-5.15.0-122-generic_5.15.0-122.132_amd64.deb ... 2024-09-17T23:32:13.627 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking linux-headers-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:13.717 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.122.122_amd64.deb ... 2024-09-17T23:32:13.839 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking linux-headers-generic (5.15.0.122.122) over (5.15.0.56.54) ... 2024-09-17T23:32:14.172 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up linux-headers-5.15.0-122 (5.15.0-122.132) ... 2024-09-17T23:32:14.293 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up linux-headers-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:14.419 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up linux-headers-generic (5.15.0.122.122) ... 2024-09-17T23:32:14.548 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up linux-image-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:15.891 INFO:teuthology.orchestra.run.smithi049.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-09-17T23:32:15.891 INFO:teuthology.orchestra.run.smithi049.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-09-17T23:32:15.891 INFO:teuthology.orchestra.run.smithi049.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-122-generic 2024-09-17T23:32:15.891 INFO:teuthology.orchestra.run.smithi049.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-122-generic 2024-09-17T23:32:16.043 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up linux-modules-extra-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:16.372 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.122.122_amd64.deb ... 2024-09-17T23:32:16.497 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking linux-headers-generic (5.15.0.122.122) over (5.15.0.56.54) ... 2024-09-17T23:32:17.864 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up linux-headers-5.15.0-122 (5.15.0-122.132) ... 2024-09-17T23:32:17.968 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up linux-headers-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:18.077 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up linux-headers-generic (5.15.0.122.122) ... 2024-09-17T23:32:18.153 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up linux-image-generic (5.15.0.122.122) ... 2024-09-17T23:32:18.198 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up linux-image-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:18.279 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up linux-generic (5.15.0.122.122) ... 2024-09-17T23:32:18.405 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up linux-modules-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:19.529 INFO:teuthology.orchestra.run.smithi028.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-09-17T23:32:19.529 INFO:teuthology.orchestra.run.smithi028.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-09-17T23:32:19.529 INFO:teuthology.orchestra.run.smithi028.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-122-generic 2024-09-17T23:32:19.530 INFO:teuthology.orchestra.run.smithi028.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-122-generic 2024-09-17T23:32:19.674 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up linux-modules-extra-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:19.832 INFO:teuthology.orchestra.run.smithi049.stdout:Processing triggers for linux-image-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:19.881 INFO:teuthology.orchestra.run.smithi049.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-09-17T23:32:19.881 INFO:teuthology.orchestra.run.smithi049.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-122-generic 2024-09-17T23:32:21.083 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up linux-image-generic (5.15.0.122.122) ... 2024-09-17T23:32:21.209 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up linux-generic (5.15.0.122.122) ... 2024-09-17T23:32:21.335 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up linux-modules-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:22.787 INFO:teuthology.orchestra.run.smithi028.stdout:Processing triggers for linux-image-5.15.0-122-generic (5.15.0-122.132) ... 2024-09-17T23:32:22.845 INFO:teuthology.orchestra.run.smithi028.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-09-17T23:32:22.845 INFO:teuthology.orchestra.run.smithi028.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-122-generic 2024-09-17T23:32:34.238 INFO:teuthology.orchestra.run.smithi049.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-09-17T23:32:34.238 INFO:teuthology.orchestra.run.smithi049.stdout:Sourcing file `/etc/default/grub' 2024-09-17T23:32:34.251 INFO:teuthology.orchestra.run.smithi049.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-17T23:32:34.274 INFO:teuthology.orchestra.run.smithi049.stdout:Generating grub configuration file ... 2024-09-17T23:32:34.550 INFO:teuthology.orchestra.run.smithi049.stdout:Found linux image: /boot/vmlinuz-5.15.0-122-generic 2024-09-17T23:32:34.566 INFO:teuthology.orchestra.run.smithi049.stdout:Found initrd image: /boot/initrd.img-5.15.0-122-generic 2024-09-17T23:32:34.853 INFO:teuthology.orchestra.run.smithi049.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-17T23:32:34.859 INFO:teuthology.orchestra.run.smithi049.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-17T23:32:34.925 INFO:teuthology.orchestra.run.smithi049.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-17T23:32:34.928 INFO:teuthology.orchestra.run.smithi049.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-17T23:32:35.086 INFO:teuthology.orchestra.run.smithi049.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-17T23:32:35.086 INFO:teuthology.orchestra.run.smithi049.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-09-17T23:32:35.086 INFO:teuthology.orchestra.run.smithi049.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-17T23:32:35.121 INFO:teuthology.orchestra.run.smithi049.stdout:done 2024-09-17T23:32:35.631 DEBUG:teuthology.orchestra.run.smithi049:> dpkg -s linux-image-generic 2024-09-17T23:32:35.649 INFO:teuthology.orchestra.run.smithi049.stdout:Package: linux-image-generic 2024-09-17T23:32:35.649 INFO:teuthology.orchestra.run.smithi049.stdout:Status: install ok installed 2024-09-17T23:32:35.649 INFO:teuthology.orchestra.run.smithi049.stdout:Priority: optional 2024-09-17T23:32:35.649 INFO:teuthology.orchestra.run.smithi049.stdout:Section: kernel 2024-09-17T23:32:35.649 INFO:teuthology.orchestra.run.smithi049.stdout:Installed-Size: 21 2024-09-17T23:32:35.649 INFO:teuthology.orchestra.run.smithi049.stdout:Maintainer: Ubuntu Kernel Team 2024-09-17T23:32:35.649 INFO:teuthology.orchestra.run.smithi049.stdout:Architecture: amd64 2024-09-17T23:32:35.649 INFO:teuthology.orchestra.run.smithi049.stdout:Source: linux-meta 2024-09-17T23:32:35.649 INFO:teuthology.orchestra.run.smithi049.stdout:Version: 5.15.0.122.122 2024-09-17T23:32:35.649 INFO:teuthology.orchestra.run.smithi049.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-122), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-17T23:32:35.649 INFO:teuthology.orchestra.run.smithi049.stdout:Depends: linux-image-5.15.0-122-generic, linux-modules-extra-5.15.0-122-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-17T23:32:35.650 INFO:teuthology.orchestra.run.smithi049.stdout:Recommends: thermald 2024-09-17T23:32:35.650 INFO:teuthology.orchestra.run.smithi049.stdout:Description: Generic Linux kernel image 2024-09-17T23:32:35.650 INFO:teuthology.orchestra.run.smithi049.stdout: This package will always depend on the latest generic kernel image 2024-09-17T23:32:35.650 INFO:teuthology.orchestra.run.smithi049.stdout: available. 2024-09-17T23:32:35.650 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-122-generic 2024-09-17T23:32:35.650 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T23:32:35.650 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T23:32:35.650 INFO:teuthology.task.kernel:Installing distro kernel on cluster2.mon.a... 2024-09-17T23:32:35.650 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi049.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T23:32:35.651 DEBUG:teuthology.orchestra.run.smithi049:> sudo apt-get clean 2024-09-17T23:32:35.812 DEBUG:teuthology.orchestra.run.smithi049:> sudo apt-get update 2024-09-17T23:32:35.925 INFO:teuthology.orchestra.run.smithi049.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-17T23:32:35.925 INFO:teuthology.orchestra.run.smithi049.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-17T23:32:35.935 INFO:teuthology.orchestra.run.smithi049.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-17T23:32:35.952 INFO:teuthology.orchestra.run.smithi049.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-17T23:32:37.223 INFO:teuthology.orchestra.run.smithi028.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-09-17T23:32:37.223 INFO:teuthology.orchestra.run.smithi028.stdout:Sourcing file `/etc/default/grub' 2024-09-17T23:32:37.239 INFO:teuthology.orchestra.run.smithi028.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-17T23:32:37.256 INFO:teuthology.orchestra.run.smithi028.stdout:Generating grub configuration file ... 2024-09-17T23:32:37.325 INFO:teuthology.orchestra.run.smithi049.stdout:Reading package lists... 2024-09-17T23:32:37.343 DEBUG:teuthology.orchestra.run.smithi049:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-17T23:32:37.409 INFO:teuthology.orchestra.run.smithi049.stdout:Reading package lists... 2024-09-17T23:32:37.510 INFO:teuthology.orchestra.run.smithi028.stdout:Found linux image: /boot/vmlinuz-5.15.0-122-generic 2024-09-17T23:32:37.524 INFO:teuthology.orchestra.run.smithi028.stdout:Found initrd image: /boot/initrd.img-5.15.0-122-generic 2024-09-17T23:32:37.625 INFO:teuthology.orchestra.run.smithi049.stdout:Building dependency tree... 2024-09-17T23:32:37.625 INFO:teuthology.orchestra.run.smithi049.stdout:Reading state information... 2024-09-17T23:32:37.812 INFO:teuthology.orchestra.run.smithi028.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-17T23:32:37.818 INFO:teuthology.orchestra.run.smithi028.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-17T23:32:37.823 INFO:teuthology.orchestra.run.smithi049.stdout:linux-image-generic is already the newest version (5.15.0.122.122). 2024-09-17T23:32:37.823 INFO:teuthology.orchestra.run.smithi049.stdout:The following packages were automatically installed and are no longer required: 2024-09-17T23:32:37.823 INFO:teuthology.orchestra.run.smithi049.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-17T23:32:37.823 INFO:teuthology.orchestra.run.smithi049.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-17T23:32:37.824 INFO:teuthology.orchestra.run.smithi049.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-17T23:32:37.824 INFO:teuthology.orchestra.run.smithi049.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-17T23:32:37.824 INFO:teuthology.orchestra.run.smithi049.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-17T23:32:37.824 INFO:teuthology.orchestra.run.smithi049.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-17T23:32:37.870 INFO:teuthology.orchestra.run.smithi049.stdout:0 upgraded, 0 newly installed, 0 to remove and 344 not upgraded. 2024-09-17T23:32:37.871 DEBUG:teuthology.orchestra.run.smithi049:> dpkg -s linux-image-generic 2024-09-17T23:32:37.885 INFO:teuthology.orchestra.run.smithi049.stdout:Package: linux-image-generic 2024-09-17T23:32:37.885 INFO:teuthology.orchestra.run.smithi049.stdout:Status: install ok installed 2024-09-17T23:32:37.885 INFO:teuthology.orchestra.run.smithi049.stdout:Priority: optional 2024-09-17T23:32:37.885 INFO:teuthology.orchestra.run.smithi049.stdout:Section: kernel 2024-09-17T23:32:37.885 INFO:teuthology.orchestra.run.smithi049.stdout:Installed-Size: 21 2024-09-17T23:32:37.885 INFO:teuthology.orchestra.run.smithi049.stdout:Maintainer: Ubuntu Kernel Team 2024-09-17T23:32:37.885 INFO:teuthology.orchestra.run.smithi049.stdout:Architecture: amd64 2024-09-17T23:32:37.885 INFO:teuthology.orchestra.run.smithi049.stdout:Source: linux-meta 2024-09-17T23:32:37.885 INFO:teuthology.orchestra.run.smithi049.stdout:Version: 5.15.0.122.122 2024-09-17T23:32:37.885 INFO:teuthology.orchestra.run.smithi049.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-122), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-17T23:32:37.886 INFO:teuthology.orchestra.run.smithi049.stdout:Depends: linux-image-5.15.0-122-generic, linux-modules-extra-5.15.0-122-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-17T23:32:37.886 INFO:teuthology.orchestra.run.smithi049.stdout:Recommends: thermald 2024-09-17T23:32:37.886 INFO:teuthology.orchestra.run.smithi049.stdout:Description: Generic Linux kernel image 2024-09-17T23:32:37.886 INFO:teuthology.orchestra.run.smithi049.stdout: This package will always depend on the latest generic kernel image 2024-09-17T23:32:37.886 INFO:teuthology.orchestra.run.smithi049.stdout: available. 2024-09-17T23:32:37.886 DEBUG:teuthology.orchestra.run.smithi049:> mktemp 2024-09-17T23:32:37.894 INFO:teuthology.orchestra.run.smithi028.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-17T23:32:37.900 INFO:teuthology.orchestra.run.smithi028.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-17T23:32:37.935 INFO:teuthology.orchestra.run.smithi049.stdout:/tmp/tmp.Xxj1eqpgQ6 2024-09-17T23:32:37.935 DEBUG:teuthology.orchestra.run.smithi049:> sudo cp /boot/grub/grub.cfg /tmp/tmp.Xxj1eqpgQ6 2024-09-17T23:32:37.991 DEBUG:teuthology.orchestra.run.smithi049:> sudo chmod 0666 /tmp/tmp.Xxj1eqpgQ6 2024-09-17T23:32:38.066 INFO:teuthology.orchestra.run.smithi028.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-17T23:32:38.066 INFO:teuthology.orchestra.run.smithi028.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-09-17T23:32:38.066 INFO:teuthology.orchestra.run.smithi028.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-17T23:32:38.114 DEBUG:teuthology.orchestra.remote:smithi049:/tmp/tmp.Xxj1eqpgQ6 is 10KB 2024-09-17T23:32:38.118 INFO:teuthology.orchestra.run.smithi028.stdout:done 2024-09-17T23:32:38.164 DEBUG:teuthology.orchestra.run.smithi049:> rm -fr /tmp/tmp.Xxj1eqpgQ6 2024-09-17T23:32:38.170 DEBUG:teuthology.orchestra.run.smithi049:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-09-17T23:32:38.223 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:32:38.223 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-09-17T23:32:38.223 DEBUG:teuthology.orchestra.run.smithi049:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-09-17T23:32:38.310 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-122-generic 2024-09-17T23:32:38.310 DEBUG:teuthology.orchestra.run.smithi049:> sudo update-grub 2024-09-17T23:32:38.498 DEBUG:teuthology.orchestra.run.smithi028:> dpkg -s linux-image-generic 2024-09-17T23:32:38.526 INFO:teuthology.orchestra.run.smithi028.stdout:Package: linux-image-generic 2024-09-17T23:32:38.526 INFO:teuthology.orchestra.run.smithi028.stdout:Status: install ok installed 2024-09-17T23:32:38.526 INFO:teuthology.orchestra.run.smithi028.stdout:Priority: optional 2024-09-17T23:32:38.527 INFO:teuthology.orchestra.run.smithi028.stdout:Section: kernel 2024-09-17T23:32:38.527 INFO:teuthology.orchestra.run.smithi028.stdout:Installed-Size: 21 2024-09-17T23:32:38.527 INFO:teuthology.orchestra.run.smithi028.stdout:Maintainer: Ubuntu Kernel Team 2024-09-17T23:32:38.527 INFO:teuthology.orchestra.run.smithi028.stdout:Architecture: amd64 2024-09-17T23:32:38.527 INFO:teuthology.orchestra.run.smithi028.stdout:Source: linux-meta 2024-09-17T23:32:38.527 INFO:teuthology.orchestra.run.smithi028.stdout:Version: 5.15.0.122.122 2024-09-17T23:32:38.527 INFO:teuthology.orchestra.run.smithi028.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-122), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-17T23:32:38.527 INFO:teuthology.orchestra.run.smithi028.stdout:Depends: linux-image-5.15.0-122-generic, linux-modules-extra-5.15.0-122-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-17T23:32:38.527 INFO:teuthology.orchestra.run.smithi028.stdout:Recommends: thermald 2024-09-17T23:32:38.527 INFO:teuthology.orchestra.run.smithi028.stdout:Description: Generic Linux kernel image 2024-09-17T23:32:38.528 INFO:teuthology.orchestra.run.smithi028.stdout: This package will always depend on the latest generic kernel image 2024-09-17T23:32:38.528 INFO:teuthology.orchestra.run.smithi028.stdout: available. 2024-09-17T23:32:38.529 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-122-generic 2024-09-17T23:32:38.529 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T23:32:38.529 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T23:32:38.529 INFO:teuthology.task.kernel:Installing distro kernel on cluster1.mon.a... 2024-09-17T23:32:38.529 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi028.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T23:32:38.529 DEBUG:teuthology.orchestra.run.smithi028:> sudo apt-get clean 2024-09-17T23:32:38.715 DEBUG:teuthology.orchestra.run.smithi028:> sudo apt-get update 2024-09-17T23:32:38.832 INFO:teuthology.orchestra.run.smithi028.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-17T23:32:38.839 INFO:teuthology.orchestra.run.smithi028.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-17T23:32:38.856 INFO:teuthology.orchestra.run.smithi028.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-17T23:32:38.952 INFO:teuthology.orchestra.run.smithi028.stdout:Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-17T23:32:39.517 INFO:teuthology.orchestra.run.smithi049.stderr:Sourcing file `/etc/default/grub' 2024-09-17T23:32:39.518 INFO:teuthology.orchestra.run.smithi049.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-17T23:32:39.520 INFO:teuthology.orchestra.run.smithi049.stderr:Generating grub configuration file ... 2024-09-17T23:32:39.712 INFO:teuthology.orchestra.run.smithi049.stderr:Found linux image: /boot/vmlinuz-5.15.0-122-generic 2024-09-17T23:32:39.718 INFO:teuthology.orchestra.run.smithi049.stderr:Found initrd image: /boot/initrd.img-5.15.0-122-generic 2024-09-17T23:32:39.945 INFO:teuthology.orchestra.run.smithi049.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-17T23:32:39.950 INFO:teuthology.orchestra.run.smithi049.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-17T23:32:39.992 INFO:teuthology.orchestra.run.smithi049.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-17T23:32:39.997 INFO:teuthology.orchestra.run.smithi049.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-17T23:32:40.109 INFO:teuthology.orchestra.run.smithi049.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-17T23:32:40.109 INFO:teuthology.orchestra.run.smithi049.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-09-17T23:32:40.109 INFO:teuthology.orchestra.run.smithi049.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-17T23:32:40.121 INFO:teuthology.orchestra.run.smithi049.stderr:done 2024-09-17T23:32:40.123 DEBUG:teuthology.orchestra.run.smithi049:> sudo shutdown -r now 2024-09-17T23:32:40.299 INFO:teuthology.orchestra.run.smithi028.stdout:Reading package lists... 2024-09-17T23:32:40.317 DEBUG:teuthology.orchestra.run.smithi028:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-17T23:32:40.381 INFO:teuthology.orchestra.run.smithi028.stdout:Reading package lists... 2024-09-17T23:32:40.603 INFO:teuthology.orchestra.run.smithi028.stdout:Building dependency tree... 2024-09-17T23:32:40.604 INFO:teuthology.orchestra.run.smithi028.stdout:Reading state information... 2024-09-17T23:32:40.803 INFO:teuthology.orchestra.run.smithi028.stdout:linux-image-generic is already the newest version (5.15.0.122.122). 2024-09-17T23:32:40.803 INFO:teuthology.orchestra.run.smithi028.stdout:The following packages were automatically installed and are no longer required: 2024-09-17T23:32:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-17T23:32:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-17T23:32:40.803 INFO:teuthology.orchestra.run.smithi028.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-17T23:32:40.804 INFO:teuthology.orchestra.run.smithi028.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-17T23:32:40.804 INFO:teuthology.orchestra.run.smithi028.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-17T23:32:40.804 INFO:teuthology.orchestra.run.smithi028.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-17T23:32:40.850 INFO:teuthology.orchestra.run.smithi028.stdout:0 upgraded, 0 newly installed, 0 to remove and 344 not upgraded. 2024-09-17T23:32:40.852 DEBUG:teuthology.orchestra.run.smithi028:> dpkg -s linux-image-generic 2024-09-17T23:32:40.865 INFO:teuthology.orchestra.run.smithi028.stdout:Package: linux-image-generic 2024-09-17T23:32:40.866 INFO:teuthology.orchestra.run.smithi028.stdout:Status: install ok installed 2024-09-17T23:32:40.866 INFO:teuthology.orchestra.run.smithi028.stdout:Priority: optional 2024-09-17T23:32:40.866 INFO:teuthology.orchestra.run.smithi028.stdout:Section: kernel 2024-09-17T23:32:40.866 INFO:teuthology.orchestra.run.smithi028.stdout:Installed-Size: 21 2024-09-17T23:32:40.866 INFO:teuthology.orchestra.run.smithi028.stdout:Maintainer: Ubuntu Kernel Team 2024-09-17T23:32:40.866 INFO:teuthology.orchestra.run.smithi028.stdout:Architecture: amd64 2024-09-17T23:32:40.866 INFO:teuthology.orchestra.run.smithi028.stdout:Source: linux-meta 2024-09-17T23:32:40.866 INFO:teuthology.orchestra.run.smithi028.stdout:Version: 5.15.0.122.122 2024-09-17T23:32:40.866 INFO:teuthology.orchestra.run.smithi028.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-122), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-17T23:32:40.866 INFO:teuthology.orchestra.run.smithi028.stdout:Depends: linux-image-5.15.0-122-generic, linux-modules-extra-5.15.0-122-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-17T23:32:40.866 INFO:teuthology.orchestra.run.smithi028.stdout:Recommends: thermald 2024-09-17T23:32:40.867 INFO:teuthology.orchestra.run.smithi028.stdout:Description: Generic Linux kernel image 2024-09-17T23:32:40.867 INFO:teuthology.orchestra.run.smithi028.stdout: This package will always depend on the latest generic kernel image 2024-09-17T23:32:40.867 INFO:teuthology.orchestra.run.smithi028.stdout: available. 2024-09-17T23:32:40.867 DEBUG:teuthology.orchestra.run.smithi028:> mktemp 2024-09-17T23:32:40.914 INFO:teuthology.orchestra.run.smithi028.stdout:/tmp/tmp.m9CPs9Mvrj 2024-09-17T23:32:40.914 DEBUG:teuthology.orchestra.run.smithi028:> sudo cp /boot/grub/grub.cfg /tmp/tmp.m9CPs9Mvrj 2024-09-17T23:32:40.970 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 0666 /tmp/tmp.m9CPs9Mvrj 2024-09-17T23:32:41.092 DEBUG:teuthology.orchestra.remote:smithi028:/tmp/tmp.m9CPs9Mvrj is 10KB 2024-09-17T23:32:41.105 DEBUG:teuthology.orchestra.run.smithi028:> rm -fr /tmp/tmp.m9CPs9Mvrj 2024-09-17T23:32:41.110 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-09-17T23:32:41.166 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:32:41.166 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-09-17T23:32:41.166 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-09-17T23:32:41.239 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-122-generic 2024-09-17T23:32:41.239 DEBUG:teuthology.orchestra.run.smithi028:> sudo update-grub 2024-09-17T23:32:42.479 INFO:teuthology.orchestra.run.smithi028.stderr:Sourcing file `/etc/default/grub' 2024-09-17T23:32:42.480 INFO:teuthology.orchestra.run.smithi028.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-17T23:32:42.482 INFO:teuthology.orchestra.run.smithi028.stderr:Generating grub configuration file ... 2024-09-17T23:32:42.649 INFO:teuthology.orchestra.run.smithi028.stderr:Found linux image: /boot/vmlinuz-5.15.0-122-generic 2024-09-17T23:32:42.656 INFO:teuthology.orchestra.run.smithi028.stderr:Found initrd image: /boot/initrd.img-5.15.0-122-generic 2024-09-17T23:32:42.888 INFO:teuthology.orchestra.run.smithi028.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-17T23:32:42.893 INFO:teuthology.orchestra.run.smithi028.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-17T23:32:42.937 INFO:teuthology.orchestra.run.smithi028.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-17T23:32:42.942 INFO:teuthology.orchestra.run.smithi028.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-17T23:32:43.064 INFO:teuthology.orchestra.run.smithi028.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-17T23:32:43.064 INFO:teuthology.orchestra.run.smithi028.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-09-17T23:32:43.064 INFO:teuthology.orchestra.run.smithi028.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-17T23:32:43.075 INFO:teuthology.orchestra.run.smithi028.stderr:done 2024-09-17T23:32:43.089 DEBUG:teuthology.orchestra.run.smithi028:> sudo shutdown -r now 2024-09-17T23:33:10.151 INFO:teuthology.task.kernel:Checking client cluster2.mon.a for new kernel version... 2024-09-17T23:33:10.152 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi049.front.sepia.ceph.com' 2024-09-17T23:33:10.153 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi049.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:33:13.095 INFO:teuthology.task.kernel:Checking client cluster1.mon.a for new kernel version... 2024-09-17T23:33:13.095 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-09-17T23:33:13.096 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:33:28.513 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.49 2024-09-17T23:33:31.585 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.28 2024-09-17T23:33:37.519 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi049.front.sepia.ceph.com' 2024-09-17T23:33:37.520 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi049.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:33:40.579 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.49 2024-09-17T23:33:40.586 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-09-17T23:33:40.586 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:33:52.589 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi049.front.sepia.ceph.com' 2024-09-17T23:33:52.589 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi049.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:33:55.645 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.49 2024-09-17T23:33:58.973 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.28 2024-09-17T23:34:10.653 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi049.front.sepia.ceph.com' 2024-09-17T23:34:10.654 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi049.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:34:10.975 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-09-17T23:34:10.975 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:34:13.729 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.49 2024-09-17T23:34:14.049 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.28 2024-09-17T23:34:29.057 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-09-17T23:34:29.058 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:34:29.350 DEBUG:teuthology.orchestra.run.smithi028:> true 2024-09-17T23:34:29.857 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-09-17T23:34:29.857 INFO:teuthology.task.kernel:Checking kernel version of cluster1.mon.a, want "5.15.0-122-generic"... 2024-09-17T23:34:29.857 DEBUG:teuthology.orchestra.run.smithi028:> uname -r 2024-09-17T23:34:29.904 INFO:teuthology.orchestra.run.smithi028.stdout:5.15.0-122-generic 2024-09-17T23:34:29.904 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-122-generic vs 5.15.0-122-generic 2024-09-17T23:34:29.905 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T23:34:29.905 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-09-17T23:34:30.905 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2024-09-17T23:34:30.906 DEBUG:teuthology.orchestra.run.smithi028:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T23:34:30.982 INFO:teuthology.orchestra.run.smithi028.stdout:ttyS1 2024-09-17T23:34:31.010 DEBUG:teuthology.parallel:result is None 2024-09-17T23:34:31.730 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi049.front.sepia.ceph.com' 2024-09-17T23:34:31.731 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi049.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:35:31.734 DEBUG:teuthology.orchestra.remote:timed out 2024-09-17T23:35:52.735 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi049.front.sepia.ceph.com' 2024-09-17T23:35:52.737 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi049.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:35:53.006 DEBUG:teuthology.orchestra.run.smithi049:> true 2024-09-17T23:35:53.564 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi049.front.sepia.ceph.com' 2024-09-17T23:35:53.565 INFO:teuthology.task.kernel:Checking kernel version of cluster2.mon.a, want "5.15.0-122-generic"... 2024-09-17T23:35:53.565 DEBUG:teuthology.orchestra.run.smithi049:> uname -r 2024-09-17T23:35:53.614 INFO:teuthology.orchestra.run.smithi049.stdout:5.15.0-122-generic 2024-09-17T23:35:53.614 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-122-generic vs 5.15.0-122-generic 2024-09-17T23:35:53.614 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T23:35:53.614 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-09-17T23:35:54.615 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2024-09-17T23:35:54.615 DEBUG:teuthology.orchestra.run.smithi049:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T23:35:54.677 INFO:teuthology.orchestra.run.smithi049.stdout:ttyS1 2024-09-17T23:35:54.702 DEBUG:teuthology.parallel:result is None 2024-09-17T23:35:54.702 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-17T23:35:54.712 INFO:teuthology.task.internal:Creating test directory... 2024-09-17T23:35:54.712 DEBUG:teuthology.orchestra.run.smithi028:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T23:35:54.715 DEBUG:teuthology.orchestra.run.smithi049:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T23:35:54.721 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-17T23:35:54.735 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-17T23:35:54.824 INFO:teuthology.task.internal:Creating archive directory... 2024-09-17T23:35:54.825 DEBUG:teuthology.orchestra.run.smithi028:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T23:35:54.827 DEBUG:teuthology.orchestra.run.smithi049:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T23:35:54.845 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-17T23:35:54.856 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-17T23:35:54.856 DEBUG:teuthology.orchestra.run.smithi028:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T23:35:54.884 DEBUG:teuthology.orchestra.run.smithi049:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T23:35:54.902 INFO:teuthology.orchestra.run.smithi028.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T23:35:54.904 INFO:teuthology.orchestra.run.smithi049.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T23:35:54.911 INFO:teuthology.orchestra.run.smithi028.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T23:35:54.913 INFO:teuthology.orchestra.run.smithi049.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T23:35:54.914 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-17T23:35:54.921 INFO:teuthology.task.internal:Configuring sudo... 2024-09-17T23:35:54.921 DEBUG:teuthology.orchestra.run.smithi028:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T23:35:54.956 DEBUG:teuthology.orchestra.run.smithi049:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T23:35:54.969 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-17T23:35:54.979 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-17T23:35:54.980 DEBUG:teuthology.orchestra.run.smithi028:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T23:35:55.012 DEBUG:teuthology.orchestra.run.smithi049:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T23:35:55.017 DEBUG:teuthology.orchestra.run.smithi028:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T23:35:55.067 DEBUG:teuthology.orchestra.run.smithi028:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T23:35:55.115 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:35:55.115 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T23:35:55.178 DEBUG:teuthology.orchestra.run.smithi049:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T23:35:55.184 DEBUG:teuthology.orchestra.run.smithi049:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T23:35:55.233 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:35:55.233 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T23:35:55.293 DEBUG:teuthology.orchestra.run.smithi028:> sudo service rsyslog restart 2024-09-17T23:35:55.296 DEBUG:teuthology.orchestra.run.smithi049:> sudo service rsyslog restart 2024-09-17T23:35:55.372 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-17T23:35:55.380 INFO:teuthology.task.internal:Starting timer... 2024-09-17T23:35:55.380 INFO:teuthology.run_tasks:Running task pcp... 2024-09-17T23:35:55.396 INFO:teuthology.run_tasks:Running task selinux... 2024-09-17T23:35:55.405 DEBUG:teuthology.task.selinux:Excluding smithi028: OS 'ubuntu' does not support SELinux 2024-09-17T23:35:55.405 DEBUG:teuthology.task.selinux:Excluding smithi049: OS 'ubuntu' does not support SELinux 2024-09-17T23:35:55.405 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-17T23:35:55.406 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2024-09-17T23:35:55.406 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-17T23:35:55.406 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-17T23:35:55.415 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-17T23:35:55.418 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-17T23:35:55.533 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-17T23:35:55.547 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-17T23:35:55.548 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi028.front.sepia.ceph.com,smithi049.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-17T23:43:35.012 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi028.front.sepia.ceph.com'), Remote(name='ubuntu@smithi049.front.sepia.ceph.com')] 2024-09-17T23:43:35.014 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-09-17T23:43:35.015 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi028.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:43:35.096 DEBUG:teuthology.orchestra.run.smithi028:> true 2024-09-17T23:43:35.181 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi028.front.sepia.ceph.com' 2024-09-17T23:43:35.181 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi049.front.sepia.ceph.com' 2024-09-17T23:43:35.182 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi049.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T23:43:35.264 DEBUG:teuthology.orchestra.run.smithi049:> true 2024-09-17T23:43:35.345 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi049.front.sepia.ceph.com' 2024-09-17T23:43:35.345 INFO:teuthology.run_tasks:Running task clock... 2024-09-17T23:43:35.356 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-17T23:43:35.356 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T23:43:35.357 DEBUG:teuthology.orchestra.run.smithi028:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T23:43:35.359 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T23:43:35.359 DEBUG:teuthology.orchestra.run.smithi049:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T23:43:35.385 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-09-17T23:43:35.385 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: Command line: ntpd -gq 2024-09-17T23:43:35.385 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: ---------------------------------------------------- 2024-09-17T23:43:35.385 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: ntp-4 is maintained by Network Time Foundation, 2024-09-17T23:43:35.386 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-09-17T23:43:35.386 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: corporation. Support and training for ntp-4 are 2024-09-17T23:43:35.386 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: available at https://www.nwtime.org/support 2024-09-17T23:43:35.386 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: ---------------------------------------------------- 2024-09-17T23:43:35.386 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: proto: precision = 0.046 usec (-24) 2024-09-17T23:43:35.386 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: basedate set to 2022-02-04 2024-09-17T23:43:35.386 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: gps base set to 2022-02-06 (week 2196) 2024-09-17T23:43:35.386 INFO:teuthology.orchestra.run.smithi028.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-17T23:43:35.387 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-17T23:43:35.387 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: restrict ::: KOD does nothing without LIMITED. 2024-09-17T23:43:35.387 INFO:teuthology.orchestra.run.smithi028.stderr:restrict ::: KOD does nothing without LIMITED. 2024-09-17T23:43:35.388 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: Listen and drop on 0 v6wildcard [::]:123 2024-09-17T23:43:35.388 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-09-17T23:43:35.388 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: Listen normally on 2 lo 127.0.0.1:123 2024-09-17T23:43:35.388 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: Listen normally on 3 ens1f0 172.21.15.28:123 2024-09-17T23:43:35.388 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: Listen normally on 4 lo [::1]:123 2024-09-17T23:43:35.388 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: Listen normally on 5 ens1f0 [fe80::ae1f:6bff:fea5:ca3c%4]:123 2024-09-17T23:43:35.388 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:35 ntpd[18396]: Listening on routing socket on fd #22 for interface updates 2024-09-17T23:43:35.418 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-09-17T23:43:35.418 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: Command line: ntpd -gq 2024-09-17T23:43:35.418 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: ---------------------------------------------------- 2024-09-17T23:43:35.418 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: ntp-4 is maintained by Network Time Foundation, 2024-09-17T23:43:35.418 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-09-17T23:43:35.418 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: corporation. Support and training for ntp-4 are 2024-09-17T23:43:35.418 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: available at https://www.nwtime.org/support 2024-09-17T23:43:35.419 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: ---------------------------------------------------- 2024-09-17T23:43:35.419 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: proto: precision = 0.068 usec (-24) 2024-09-17T23:43:35.419 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: basedate set to 2022-02-04 2024-09-17T23:43:35.419 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: gps base set to 2022-02-06 (week 2196) 2024-09-17T23:43:35.420 INFO:teuthology.orchestra.run.smithi049.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-17T23:43:35.420 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-17T23:43:35.420 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: restrict ::: KOD does nothing without LIMITED. 2024-09-17T23:43:35.420 INFO:teuthology.orchestra.run.smithi049.stderr:restrict ::: KOD does nothing without LIMITED. 2024-09-17T23:43:35.421 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: Listen and drop on 0 v6wildcard [::]:123 2024-09-17T23:43:35.421 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-09-17T23:43:35.421 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: Listen normally on 2 lo 127.0.0.1:123 2024-09-17T23:43:35.421 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: Listen normally on 3 ens1f0 172.21.15.49:123 2024-09-17T23:43:35.421 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: Listen normally on 4 lo [::1]:123 2024-09-17T23:43:35.422 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: Listen normally on 5 ens1f0 [fe80::ec4:7aff:febd:1566%4]:123 2024-09-17T23:43:35.422 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:35 ntpd[18382]: Listening on routing socket on fd #22 for interface updates 2024-09-17T23:43:36.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:36.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:36.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:36.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:36.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:36.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:36.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:36.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:36.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:36.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:36.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:36.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:36.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:36.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:36.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:36.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:36.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:36.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:36.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:36.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:36 ntpd[18396]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:36.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:36 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:36.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:36 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:36.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:36 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:36.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:36 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:36.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:36 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:36.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:36 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:36.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:36 ntpd[18382]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:36.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:36 ntpd[18382]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:38.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:38.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:38.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:38.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:38.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:38.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:38.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:38.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:38.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:38.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:38.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:38.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:38.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:38.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:38.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:38.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:38.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:38.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:38 ntpd[18396]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:38.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:38 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:38.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:38 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:38.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:38 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:38.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:38 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:38.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:38 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:38.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:38 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:38.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:38 ntpd[18382]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:38.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:38 ntpd[18382]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:38.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:38 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:38.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:38 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:38.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:38 ntpd[18382]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:38.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:38 ntpd[18382]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:40.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:40.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:40.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:40.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:40.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:40.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:40.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:40.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:40.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:40.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:40.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:40.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:40.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:40.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:40.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:40.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:40.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:40.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:40 ntpd[18396]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:40.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:40.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:40.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:40.421 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:40.421 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:40.421 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:40 ntpd[18382]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:42.386 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:42 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:42.386 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:42 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:42.386 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:42 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:42.386 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:42 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:42.386 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:42 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:42.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:42 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:42.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:42 ntpd[18396]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:42.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:42 ntpd[18396]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:42.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:42 ntpd[18396]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:42.387 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:42 ntpd[18396]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:42.387 INFO:teuthology.orchestra.run.smithi028.stdout:17 Sep 23:43:42 ntpd[18396]: ntpd: time slew -0.003056 s 2024-09-17T23:43:42.387 INFO:teuthology.orchestra.run.smithi028.stdout:ntpd: time slew -0.003056s 2024-09-17T23:43:42.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:42 ntpd[18396]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-09-17T23:43:42.388 INFO:teuthology.orchestra.run.smithi028.stderr:17 Sep 23:43:42 ntpd[18396]: can't open /var/log/ntpstats/loopstats.20240917: Permission denied 2024-09-17T23:43:42.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:42 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:42.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:42 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:42.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:42 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:42.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:42 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:42.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:42 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:42.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:42 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:42.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:42 ntpd[18382]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-17T23:43:42.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:42 ntpd[18382]: can't open /var/log/ntpstats/rawstats.20240917: Permission denied 2024-09-17T23:43:42.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:42 ntpd[18382]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-17T23:43:42.419 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:42 ntpd[18382]: can't open /var/log/ntpstats/peerstats.20240917: Permission denied 2024-09-17T23:43:42.420 INFO:teuthology.orchestra.run.smithi049.stdout:17 Sep 23:43:42 ntpd[18382]: ntpd: time slew -0.001236 s 2024-09-17T23:43:42.420 INFO:teuthology.orchestra.run.smithi049.stdout:ntpd: time slew -0.001236s 2024-09-17T23:43:42.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:42 ntpd[18382]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-09-17T23:43:42.420 INFO:teuthology.orchestra.run.smithi049.stderr:17 Sep 23:43:42 ntpd[18382]: can't open /var/log/ntpstats/loopstats.20240917: Permission denied 2024-09-17T23:43:42.437 INFO:teuthology.orchestra.run.smithi028.stdout: remote refid st t when poll reach delay offset jitter 2024-09-17T23:43:42.437 INFO:teuthology.orchestra.run.smithi028.stdout:============================================================================== 2024-09-17T23:43:42.437 INFO:teuthology.orchestra.run.smithi028.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-17T23:43:42.437 INFO:teuthology.orchestra.run.smithi028.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-17T23:43:42.437 INFO:teuthology.orchestra.run.smithi028.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-17T23:43:42.437 INFO:teuthology.orchestra.run.smithi028.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-17T23:43:42.471 INFO:teuthology.orchestra.run.smithi049.stdout: remote refid st t when poll reach delay offset jitter 2024-09-17T23:43:42.471 INFO:teuthology.orchestra.run.smithi049.stdout:============================================================================== 2024-09-17T23:43:42.471 INFO:teuthology.orchestra.run.smithi049.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-17T23:43:42.471 INFO:teuthology.orchestra.run.smithi049.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-17T23:43:42.471 INFO:teuthology.orchestra.run.smithi049.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-17T23:43:42.471 INFO:teuthology.orchestra.run.smithi049.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-17T23:43:42.472 INFO:teuthology.run_tasks:Running task install... 2024-09-17T23:43:42.480 DEBUG:teuthology.task.install:project ceph 2024-09-17T23:43:42.481 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'extra_packages': ['rbd-nbd'], 'flavor': 'default', 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}, 'extra_system_packages': ['pv']} 2024-09-17T23:43:42.481 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror', 'rbd-nbd'], 'flavor': 'default', 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f', 'extra_system_packages': ['pv']} 2024-09-17T23:43:42.481 INFO:teuthology.task.install:Using flavor: default 2024-09-17T23:43:42.486 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-17T23:43:42.486 INFO:teuthology.task.install:extra packages: ['rbd-mirror', 'rbd-nbd'] 2024-09-17T23:43:42.487 DEBUG:teuthology.orchestra.run.smithi028:> sudo apt-key list | grep Ceph 2024-09-17T23:43:42.487 DEBUG:teuthology.orchestra.run.smithi049:> sudo apt-key list | grep Ceph 2024-09-17T23:43:42.548 INFO:teuthology.orchestra.run.smithi028.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-09-17T23:43:42.571 INFO:teuthology.orchestra.run.smithi049.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-09-17T23:43:42.580 INFO:teuthology.orchestra.run.smithi028.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-09-17T23:43:42.580 INFO:teuthology.orchestra.run.smithi028.stdout:uid [ unknown] Ceph.com (release key) 2024-09-17T23:43:42.580 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror, rbd-nbd on remote deb x86_64 2024-09-17T23:43:42.580 INFO:teuthology.task.install.deb:Installing system (non-project) packages: pv on remote deb x86_64 2024-09-17T23:43:42.581 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T23:43:42.603 INFO:teuthology.orchestra.run.smithi049.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-09-17T23:43:42.603 INFO:teuthology.orchestra.run.smithi049.stdout:uid [ unknown] Ceph.com (release key) 2024-09-17T23:43:42.603 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror, rbd-nbd on remote deb x86_64 2024-09-17T23:43:42.604 INFO:teuthology.task.install.deb:Installing system (non-project) packages: pv on remote deb x86_64 2024-09-17T23:43:42.604 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T23:43:42.723 INFO:teuthology.task.install.deb:Pulling from https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default/ 2024-09-17T23:43:42.723 INFO:teuthology.task.install.deb:Package version is 19.1.1-330-gc2ddf972-1jammy 2024-09-17T23:43:42.742 INFO:teuthology.task.install.deb:Pulling from https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default/ 2024-09-17T23:43:42.742 INFO:teuthology.task.install.deb:Package version is 19.1.1-330-gc2ddf972-1jammy 2024-09-17T23:43:42.837 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:43:42.837 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-09-17T23:43:42.852 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:43:42.852 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-09-17T23:43:42.853 DEBUG:teuthology.orchestra.run.smithi028:> sudo apt-get update 2024-09-17T23:43:42.867 DEBUG:teuthology.orchestra.run.smithi049:> sudo apt-get update 2024-09-17T23:43:43.022 INFO:teuthology.orchestra.run.smithi028.stdout:Hit:1 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-17T23:43:43.039 INFO:teuthology.orchestra.run.smithi049.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-17T23:43:43.040 INFO:teuthology.orchestra.run.smithi049.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-17T23:43:43.048 INFO:teuthology.orchestra.run.smithi049.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-17T23:43:43.066 INFO:teuthology.orchestra.run.smithi049.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-17T23:43:43.088 INFO:teuthology.orchestra.run.smithi028.stdout:Ign:2 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy InRelease 2024-09-17T23:43:43.093 INFO:teuthology.orchestra.run.smithi049.stdout:Ign:5 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy InRelease 2024-09-17T23:43:43.109 INFO:teuthology.orchestra.run.smithi028.stdout:Get:3 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy Release [7,662 B] 2024-09-17T23:43:43.115 INFO:teuthology.orchestra.run.smithi049.stdout:Get:6 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy Release [7,662 B] 2024-09-17T23:43:43.131 INFO:teuthology.orchestra.run.smithi028.stdout:Ign:4 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy Release.gpg 2024-09-17T23:43:43.137 INFO:teuthology.orchestra.run.smithi049.stdout:Ign:7 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy Release.gpg 2024-09-17T23:43:43.142 INFO:teuthology.orchestra.run.smithi028.stdout:Hit:5 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-17T23:43:43.152 INFO:teuthology.orchestra.run.smithi028.stdout:Get:6 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.9 kB] 2024-09-17T23:43:43.159 INFO:teuthology.orchestra.run.smithi049.stdout:Get:8 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.9 kB] 2024-09-17T23:43:43.180 INFO:teuthology.orchestra.run.smithi028.stdout:Get:7 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,409 B] 2024-09-17T23:43:43.187 INFO:teuthology.orchestra.run.smithi049.stdout:Get:9 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,409 B] 2024-09-17T23:43:43.224 INFO:teuthology.orchestra.run.smithi028.stdout:Hit:8 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-17T23:43:43.305 INFO:teuthology.orchestra.run.smithi028.stdout:Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-17T23:43:43.427 INFO:teuthology.orchestra.run.smithi049.stdout:Fetched 28.9 kB in 0s (76.5 kB/s) 2024-09-17T23:43:43.446 INFO:teuthology.orchestra.run.smithi028.stdout:Fetched 28.9 kB in 0s (70.4 kB/s) 2024-09-17T23:43:44.464 INFO:teuthology.orchestra.run.smithi049.stdout:Reading package lists... 2024-09-17T23:43:44.481 DEBUG:teuthology.orchestra.run.smithi049:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=19.1.1-330-gc2ddf972-1jammy cephadm=19.1.1-330-gc2ddf972-1jammy ceph-mds=19.1.1-330-gc2ddf972-1jammy ceph-mgr=19.1.1-330-gc2ddf972-1jammy ceph-common=19.1.1-330-gc2ddf972-1jammy ceph-fuse=19.1.1-330-gc2ddf972-1jammy ceph-test=19.1.1-330-gc2ddf972-1jammy ceph-volume=19.1.1-330-gc2ddf972-1jammy radosgw=19.1.1-330-gc2ddf972-1jammy python3-rados=19.1.1-330-gc2ddf972-1jammy python3-rgw=19.1.1-330-gc2ddf972-1jammy python3-cephfs=19.1.1-330-gc2ddf972-1jammy python3-rbd=19.1.1-330-gc2ddf972-1jammy libcephfs2=19.1.1-330-gc2ddf972-1jammy libcephfs-dev=19.1.1-330-gc2ddf972-1jammy librados2=19.1.1-330-gc2ddf972-1jammy librbd1=19.1.1-330-gc2ddf972-1jammy rbd-fuse=19.1.1-330-gc2ddf972-1jammy rbd-mirror=19.1.1-330-gc2ddf972-1jammy rbd-nbd=19.1.1-330-gc2ddf972-1jammy 2024-09-17T23:43:44.486 INFO:teuthology.orchestra.run.smithi028.stdout:Reading package lists... 2024-09-17T23:43:44.501 DEBUG:teuthology.orchestra.run.smithi028:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=19.1.1-330-gc2ddf972-1jammy cephadm=19.1.1-330-gc2ddf972-1jammy ceph-mds=19.1.1-330-gc2ddf972-1jammy ceph-mgr=19.1.1-330-gc2ddf972-1jammy ceph-common=19.1.1-330-gc2ddf972-1jammy ceph-fuse=19.1.1-330-gc2ddf972-1jammy ceph-test=19.1.1-330-gc2ddf972-1jammy ceph-volume=19.1.1-330-gc2ddf972-1jammy radosgw=19.1.1-330-gc2ddf972-1jammy python3-rados=19.1.1-330-gc2ddf972-1jammy python3-rgw=19.1.1-330-gc2ddf972-1jammy python3-cephfs=19.1.1-330-gc2ddf972-1jammy python3-rbd=19.1.1-330-gc2ddf972-1jammy libcephfs2=19.1.1-330-gc2ddf972-1jammy libcephfs-dev=19.1.1-330-gc2ddf972-1jammy librados2=19.1.1-330-gc2ddf972-1jammy librbd1=19.1.1-330-gc2ddf972-1jammy rbd-fuse=19.1.1-330-gc2ddf972-1jammy rbd-mirror=19.1.1-330-gc2ddf972-1jammy rbd-nbd=19.1.1-330-gc2ddf972-1jammy 2024-09-17T23:43:44.536 INFO:teuthology.orchestra.run.smithi049.stdout:Reading package lists... 2024-09-17T23:43:44.555 INFO:teuthology.orchestra.run.smithi028.stdout:Reading package lists... 2024-09-17T23:43:44.733 INFO:teuthology.orchestra.run.smithi049.stdout:Building dependency tree... 2024-09-17T23:43:44.734 INFO:teuthology.orchestra.run.smithi049.stdout:Reading state information... 2024-09-17T23:43:44.745 INFO:teuthology.orchestra.run.smithi028.stdout:Building dependency tree... 2024-09-17T23:43:44.746 INFO:teuthology.orchestra.run.smithi028.stdout:Reading state information... 2024-09-17T23:43:44.929 INFO:teuthology.orchestra.run.smithi049.stdout:The following packages were automatically installed and are no longer required: 2024-09-17T23:43:44.930 INFO:teuthology.orchestra.run.smithi049.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-17T23:43:44.930 INFO:teuthology.orchestra.run.smithi049.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-17T23:43:44.930 INFO:teuthology.orchestra.run.smithi049.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-17T23:43:44.930 INFO:teuthology.orchestra.run.smithi049.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-17T23:43:44.930 INFO:teuthology.orchestra.run.smithi049.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-17T23:43:44.930 INFO:teuthology.orchestra.run.smithi049.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-17T23:43:44.931 INFO:teuthology.orchestra.run.smithi049.stdout:The following additional packages will be installed: 2024-09-17T23:43:44.931 INFO:teuthology.orchestra.run.smithi049.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-09-17T23:43:44.931 INFO:teuthology.orchestra.run.smithi049.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2024-09-17T23:43:44.931 INFO:teuthology.orchestra.run.smithi049.stdout: libdouble-conversion3 libjq1 liblttng-ust1 liblua5.3-dev liboath0 libonig5 2024-09-17T23:43:44.931 INFO:teuthology.orchestra.run.smithi049.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: librdkafka1 libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: lua-any lua-sec lua-socket lua5.1 luarocks nvme-cli pkg-config 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: python-asyncssh-doc python-babel-localedata python-pastedeploy-tpl 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: python3-asyncssh python3-babel python3-bcrypt python3-cachetools 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: python3-ceph-argparse python3-ceph-common python3-certifi python3-cheroot 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-09-17T23:43:44.933 INFO:teuthology.orchestra.run.smithi049.stdout: python3-pygments python3-pyinotify python3-pytest python3-repoze.lru 2024-09-17T23:43:44.934 INFO:teuthology.orchestra.run.smithi049.stdout: python3-requests python3-requests-oauthlib python3-routes python3-rsa 2024-09-17T23:43:44.934 INFO:teuthology.orchestra.run.smithi049.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-09-17T23:43:44.934 INFO:teuthology.orchestra.run.smithi049.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-09-17T23:43:44.934 INFO:teuthology.orchestra.run.smithi049.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-09-17T23:43:44.934 INFO:teuthology.orchestra.run.smithi049.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-09-17T23:43:44.934 INFO:teuthology.orchestra.run.smithi049.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat unzip 2024-09-17T23:43:44.934 INFO:teuthology.orchestra.run.smithi049.stdout: xmlstarlet zip 2024-09-17T23:43:44.934 INFO:teuthology.orchestra.run.smithi049.stdout:Suggested packages: 2024-09-17T23:43:44.934 INFO:teuthology.orchestra.run.smithi049.stdout: python3-influxdb readline-doc python-jinja2-doc python3-beaker 2024-09-17T23:43:44.935 INFO:teuthology.orchestra.run.smithi049.stdout: python-mako-doc python-natsort-doc python-openssl-doc python3-openssl-dbg 2024-09-17T23:43:44.935 INFO:teuthology.orchestra.run.smithi049.stdout: httpd-wsgi libapache2-mod-python libapache2-mod-scgi libjs-mochikit 2024-09-17T23:43:44.935 INFO:teuthology.orchestra.run.smithi049.stdout: python-pecan-doc python-psutil-doc subversion python-pygments-doc 2024-09-17T23:43:44.935 INFO:teuthology.orchestra.run.smithi049.stdout: python-pyinotify-doc python3-socks python-requests-doc python3-dap 2024-09-17T23:43:44.935 INFO:teuthology.orchestra.run.smithi049.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2024-09-17T23:43:44.935 INFO:teuthology.orchestra.run.smithi049.stdout: python-webtest-doc python-werkzeug-doc python3-watchdog 2024-09-17T23:43:44.935 INFO:teuthology.orchestra.run.smithi049.stdout:Recommended packages: 2024-09-17T23:43:44.935 INFO:teuthology.orchestra.run.smithi049.stdout: btrfs-tools 2024-09-17T23:43:44.938 INFO:teuthology.orchestra.run.smithi028.stdout:The following packages were automatically installed and are no longer required: 2024-09-17T23:43:44.938 INFO:teuthology.orchestra.run.smithi028.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-17T23:43:44.938 INFO:teuthology.orchestra.run.smithi028.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-17T23:43:44.938 INFO:teuthology.orchestra.run.smithi028.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-17T23:43:44.939 INFO:teuthology.orchestra.run.smithi028.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-17T23:43:44.939 INFO:teuthology.orchestra.run.smithi028.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-17T23:43:44.939 INFO:teuthology.orchestra.run.smithi028.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-17T23:43:44.939 INFO:teuthology.orchestra.run.smithi028.stdout:The following additional packages will be installed: 2024-09-17T23:43:44.939 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-09-17T23:43:44.940 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2024-09-17T23:43:44.940 INFO:teuthology.orchestra.run.smithi028.stdout: libdouble-conversion3 libjq1 liblttng-ust1 liblua5.3-dev liboath0 libonig5 2024-09-17T23:43:44.940 INFO:teuthology.orchestra.run.smithi028.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-09-17T23:43:44.941 INFO:teuthology.orchestra.run.smithi028.stdout: librdkafka1 libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0 2024-09-17T23:43:44.941 INFO:teuthology.orchestra.run.smithi028.stdout: lua-any lua-sec lua-socket lua5.1 luarocks nvme-cli pkg-config 2024-09-17T23:43:44.941 INFO:teuthology.orchestra.run.smithi028.stdout: python-asyncssh-doc python-babel-localedata python-pastedeploy-tpl 2024-09-17T23:43:44.941 INFO:teuthology.orchestra.run.smithi028.stdout: python3-asyncssh python3-babel python3-bcrypt python3-cachetools 2024-09-17T23:43:44.941 INFO:teuthology.orchestra.run.smithi028.stdout: python3-ceph-argparse python3-ceph-common python3-certifi python3-cheroot 2024-09-17T23:43:44.941 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-09-17T23:43:44.941 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-09-17T23:43:44.942 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-09-17T23:43:44.942 INFO:teuthology.orchestra.run.smithi028.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-09-17T23:43:44.942 INFO:teuthology.orchestra.run.smithi028.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-09-17T23:43:44.942 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-09-17T23:43:44.942 INFO:teuthology.orchestra.run.smithi028.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-09-17T23:43:44.942 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pygments python3-pyinotify python3-pytest python3-repoze.lru 2024-09-17T23:43:44.942 INFO:teuthology.orchestra.run.smithi028.stdout: python3-requests python3-requests-oauthlib python3-routes python3-rsa 2024-09-17T23:43:44.942 INFO:teuthology.orchestra.run.smithi028.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-09-17T23:43:44.942 INFO:teuthology.orchestra.run.smithi028.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-09-17T23:43:44.942 INFO:teuthology.orchestra.run.smithi028.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-09-17T23:43:44.942 INFO:teuthology.orchestra.run.smithi028.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-09-17T23:43:44.942 INFO:teuthology.orchestra.run.smithi028.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat unzip 2024-09-17T23:43:44.943 INFO:teuthology.orchestra.run.smithi028.stdout: xmlstarlet zip 2024-09-17T23:43:44.943 INFO:teuthology.orchestra.run.smithi028.stdout:Suggested packages: 2024-09-17T23:43:44.943 INFO:teuthology.orchestra.run.smithi028.stdout: python3-influxdb readline-doc python-jinja2-doc python3-beaker 2024-09-17T23:43:44.943 INFO:teuthology.orchestra.run.smithi028.stdout: python-mako-doc python-natsort-doc python-openssl-doc python3-openssl-dbg 2024-09-17T23:43:44.943 INFO:teuthology.orchestra.run.smithi028.stdout: httpd-wsgi libapache2-mod-python libapache2-mod-scgi libjs-mochikit 2024-09-17T23:43:44.943 INFO:teuthology.orchestra.run.smithi028.stdout: python-pecan-doc python-psutil-doc subversion python-pygments-doc 2024-09-17T23:43:44.943 INFO:teuthology.orchestra.run.smithi028.stdout: python-pyinotify-doc python3-socks python-requests-doc python3-dap 2024-09-17T23:43:44.944 INFO:teuthology.orchestra.run.smithi028.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2024-09-17T23:43:44.944 INFO:teuthology.orchestra.run.smithi028.stdout: python-webtest-doc python-werkzeug-doc python3-watchdog 2024-09-17T23:43:44.944 INFO:teuthology.orchestra.run.smithi028.stdout:Recommended packages: 2024-09-17T23:43:44.944 INFO:teuthology.orchestra.run.smithi028.stdout: btrfs-tools 2024-09-17T23:43:45.043 INFO:teuthology.orchestra.run.smithi049.stdout:The following NEW packages will be installed: 2024-09-17T23:43:45.043 INFO:teuthology.orchestra.run.smithi049.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-09-17T23:43:45.043 INFO:teuthology.orchestra.run.smithi049.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-09-17T23:43:45.044 INFO:teuthology.orchestra.run.smithi049.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-09-17T23:43:45.044 INFO:teuthology.orchestra.run.smithi049.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libjq1 liblttng-ust1 2024-09-17T23:43:45.044 INFO:teuthology.orchestra.run.smithi049.stdout: liblua5.3-dev liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 2024-09-17T23:43:45.044 INFO:teuthology.orchestra.run.smithi049.stdout: libqt5network5 librados2 libradosstriper1 librbd1 librdkafka1 2024-09-17T23:43:45.045 INFO:teuthology.orchestra.run.smithi049.stdout: libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0 lua-any lua-sec 2024-09-17T23:43:45.045 INFO:teuthology.orchestra.run.smithi049.stdout: lua-socket lua5.1 luarocks nvme-cli pkg-config python-asyncssh-doc 2024-09-17T23:43:45.045 INFO:teuthology.orchestra.run.smithi049.stdout: python-babel-localedata python-pastedeploy-tpl python3-asyncssh 2024-09-17T23:43:45.045 INFO:teuthology.orchestra.run.smithi049.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-09-17T23:43:45.045 INFO:teuthology.orchestra.run.smithi049.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-09-17T23:43:45.045 INFO:teuthology.orchestra.run.smithi049.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-09-17T23:43:45.045 INFO:teuthology.orchestra.run.smithi049.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-09-17T23:43:45.045 INFO:teuthology.orchestra.run.smithi049.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-09-17T23:43:45.046 INFO:teuthology.orchestra.run.smithi049.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-09-17T23:43:45.046 INFO:teuthology.orchestra.run.smithi049.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-09-17T23:43:45.046 INFO:teuthology.orchestra.run.smithi049.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-09-17T23:43:45.046 INFO:teuthology.orchestra.run.smithi049.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-09-17T23:43:45.046 INFO:teuthology.orchestra.run.smithi049.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-09-17T23:43:45.046 INFO:teuthology.orchestra.run.smithi049.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-09-17T23:43:45.046 INFO:teuthology.orchestra.run.smithi049.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-09-17T23:43:45.046 INFO:teuthology.orchestra.run.smithi049.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-09-17T23:43:45.046 INFO:teuthology.orchestra.run.smithi049.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-09-17T23:43:45.046 INFO:teuthology.orchestra.run.smithi049.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-09-17T23:43:45.046 INFO:teuthology.orchestra.run.smithi049.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-09-17T23:43:45.046 INFO:teuthology.orchestra.run.smithi049.stdout: radosgw rbd-fuse rbd-mirror rbd-nbd socat unzip xmlstarlet zip 2024-09-17T23:43:45.050 INFO:teuthology.orchestra.run.smithi028.stdout:The following NEW packages will be installed: 2024-09-17T23:43:45.050 INFO:teuthology.orchestra.run.smithi028.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-09-17T23:43:45.050 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-09-17T23:43:45.050 INFO:teuthology.orchestra.run.smithi028.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-09-17T23:43:45.052 INFO:teuthology.orchestra.run.smithi028.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libjq1 liblttng-ust1 2024-09-17T23:43:45.052 INFO:teuthology.orchestra.run.smithi028.stdout: liblua5.3-dev liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 2024-09-17T23:43:45.052 INFO:teuthology.orchestra.run.smithi028.stdout: libqt5network5 librados2 libradosstriper1 librbd1 librdkafka1 2024-09-17T23:43:45.052 INFO:teuthology.orchestra.run.smithi028.stdout: libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0 lua-any lua-sec 2024-09-17T23:43:45.052 INFO:teuthology.orchestra.run.smithi028.stdout: lua-socket lua5.1 luarocks nvme-cli pkg-config python-asyncssh-doc 2024-09-17T23:43:45.052 INFO:teuthology.orchestra.run.smithi028.stdout: python-babel-localedata python-pastedeploy-tpl python3-asyncssh 2024-09-17T23:43:45.052 INFO:teuthology.orchestra.run.smithi028.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-09-17T23:43:45.052 INFO:teuthology.orchestra.run.smithi028.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-09-17T23:43:45.052 INFO:teuthology.orchestra.run.smithi028.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-09-17T23:43:45.052 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-09-17T23:43:45.052 INFO:teuthology.orchestra.run.smithi028.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-09-17T23:43:45.052 INFO:teuthology.orchestra.run.smithi028.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-09-17T23:43:45.053 INFO:teuthology.orchestra.run.smithi028.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-09-17T23:43:45.053 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-09-17T23:43:45.053 INFO:teuthology.orchestra.run.smithi028.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-09-17T23:43:45.053 INFO:teuthology.orchestra.run.smithi028.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-09-17T23:43:45.053 INFO:teuthology.orchestra.run.smithi028.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-09-17T23:43:45.053 INFO:teuthology.orchestra.run.smithi028.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-09-17T23:43:45.053 INFO:teuthology.orchestra.run.smithi028.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-09-17T23:43:45.053 INFO:teuthology.orchestra.run.smithi028.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-09-17T23:43:45.053 INFO:teuthology.orchestra.run.smithi028.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-09-17T23:43:45.053 INFO:teuthology.orchestra.run.smithi028.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-09-17T23:43:45.053 INFO:teuthology.orchestra.run.smithi028.stdout: radosgw rbd-fuse rbd-mirror rbd-nbd socat unzip xmlstarlet zip 2024-09-17T23:43:45.192 INFO:teuthology.orchestra.run.smithi049.stdout:0 upgraded, 120 newly installed, 0 to remove and 331 not upgraded. 2024-09-17T23:43:45.192 INFO:teuthology.orchestra.run.smithi049.stdout:Need to get 184 MB of archives. 2024-09-17T23:43:45.192 INFO:teuthology.orchestra.run.smithi049.stdout:After this operation, 843 MB of additional disk space will be used. 2024-09-17T23:43:45.192 INFO:teuthology.orchestra.run.smithi049.stdout:Get:1 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 19.1.1-330-gc2ddf972-1jammy [3,591 kB] 2024-09-17T23:43:45.198 INFO:teuthology.orchestra.run.smithi028.stdout:0 upgraded, 120 newly installed, 0 to remove and 331 not upgraded. 2024-09-17T23:43:45.198 INFO:teuthology.orchestra.run.smithi028.stdout:Need to get 184 MB of archives. 2024-09-17T23:43:45.198 INFO:teuthology.orchestra.run.smithi028.stdout:After this operation, 843 MB of additional disk space will be used. 2024-09-17T23:43:45.198 INFO:teuthology.orchestra.run.smithi028.stdout:Get:1 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 19.1.1-330-gc2ddf972-1jammy [3,591 kB] 2024-09-17T23:43:45.236 INFO:teuthology.orchestra.run.smithi049.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-09-17T23:43:45.241 INFO:teuthology.orchestra.run.smithi028.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-09-17T23:43:45.400 INFO:teuthology.orchestra.run.smithi049.stdout:Get:3 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 19.1.1-330-gc2ddf972-1jammy [3,242 kB] 2024-09-17T23:43:45.404 INFO:teuthology.orchestra.run.smithi028.stdout:Get:3 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 19.1.1-330-gc2ddf972-1jammy [3,242 kB] 2024-09-17T23:43:45.491 INFO:teuthology.orchestra.run.smithi049.stdout:Get:4 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 19.1.1-330-gc2ddf972-1jammy [719 kB] 2024-09-17T23:43:45.508 INFO:teuthology.orchestra.run.smithi028.stdout:Get:4 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 19.1.1-330-gc2ddf972-1jammy [719 kB] 2024-09-17T23:43:45.518 INFO:teuthology.orchestra.run.smithi049.stdout:Get:5 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 19.1.1-330-gc2ddf972-1jammy [356 kB] 2024-09-17T23:43:45.525 INFO:teuthology.orchestra.run.smithi049.stdout:Get:6 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 19.1.1-330-gc2ddf972-1jammy [32.8 kB] 2024-09-17T23:43:45.526 INFO:teuthology.orchestra.run.smithi049.stdout:Get:7 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 19.1.1-330-gc2ddf972-1jammy [180 kB] 2024-09-17T23:43:45.530 INFO:teuthology.orchestra.run.smithi049.stdout:Get:8 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 19.1.1-330-gc2ddf972-1jammy [67.8 kB] 2024-09-17T23:43:45.532 INFO:teuthology.orchestra.run.smithi049.stdout:Get:9 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 19.1.1-330-gc2ddf972-1jammy [333 kB] 2024-09-17T23:43:45.535 INFO:teuthology.orchestra.run.smithi028.stdout:Get:5 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 19.1.1-330-gc2ddf972-1jammy [356 kB] 2024-09-17T23:43:45.542 INFO:teuthology.orchestra.run.smithi028.stdout:Get:6 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 19.1.1-330-gc2ddf972-1jammy [32.8 kB] 2024-09-17T23:43:45.543 INFO:teuthology.orchestra.run.smithi028.stdout:Get:7 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 19.1.1-330-gc2ddf972-1jammy [180 kB] 2024-09-17T23:43:45.547 INFO:teuthology.orchestra.run.smithi028.stdout:Get:8 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 19.1.1-330-gc2ddf972-1jammy [67.8 kB] 2024-09-17T23:43:45.549 INFO:teuthology.orchestra.run.smithi028.stdout:Get:9 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 19.1.1-330-gc2ddf972-1jammy [333 kB] 2024-09-17T23:43:45.550 INFO:teuthology.orchestra.run.smithi049.stdout:Get:10 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 19.1.1-330-gc2ddf972-1jammy [6,875 kB] 2024-09-17T23:43:45.568 INFO:teuthology.orchestra.run.smithi028.stdout:Get:10 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 19.1.1-330-gc2ddf972-1jammy [6,875 kB] 2024-09-17T23:43:45.659 INFO:teuthology.orchestra.run.smithi049.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-09-17T23:43:45.664 INFO:teuthology.orchestra.run.smithi028.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-09-17T23:43:45.672 INFO:teuthology.orchestra.run.smithi049.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-09-17T23:43:45.677 INFO:teuthology.orchestra.run.smithi028.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-09-17T23:43:45.753 INFO:teuthology.orchestra.run.smithi049.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-09-17T23:43:45.758 INFO:teuthology.orchestra.run.smithi028.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-09-17T23:43:45.766 INFO:teuthology.orchestra.run.smithi049.stdout:Get:14 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 19.1.1-330-gc2ddf972-1jammy [112 kB] 2024-09-17T23:43:45.768 INFO:teuthology.orchestra.run.smithi049.stdout:Get:15 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 19.1.1-330-gc2ddf972-1jammy [472 kB] 2024-09-17T23:43:45.790 INFO:teuthology.orchestra.run.smithi028.stdout:Get:14 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 19.1.1-330-gc2ddf972-1jammy [112 kB] 2024-09-17T23:43:45.792 INFO:teuthology.orchestra.run.smithi028.stdout:Get:15 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 19.1.1-330-gc2ddf972-1jammy [472 kB] 2024-09-17T23:43:45.795 INFO:teuthology.orchestra.run.smithi049.stdout:Get:16 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 19.1.1-330-gc2ddf972-1jammy [25.9 MB] 2024-09-17T23:43:45.819 INFO:teuthology.orchestra.run.smithi028.stdout:Get:16 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 19.1.1-330-gc2ddf972-1jammy [25.9 MB] 2024-09-17T23:43:45.982 INFO:teuthology.orchestra.run.smithi049.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-09-17T23:43:45.987 INFO:teuthology.orchestra.run.smithi028.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-09-17T23:43:45.992 INFO:teuthology.orchestra.run.smithi049.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-09-17T23:43:45.997 INFO:teuthology.orchestra.run.smithi028.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-09-17T23:43:46.022 INFO:teuthology.orchestra.run.smithi049.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-09-17T23:43:46.027 INFO:teuthology.orchestra.run.smithi028.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-09-17T23:43:46.031 INFO:teuthology.orchestra.run.smithi049.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-09-17T23:43:46.032 INFO:teuthology.orchestra.run.smithi049.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-09-17T23:43:46.033 INFO:teuthology.orchestra.run.smithi049.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-09-17T23:43:46.036 INFO:teuthology.orchestra.run.smithi028.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-09-17T23:43:46.037 INFO:teuthology.orchestra.run.smithi028.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-09-17T23:43:46.038 INFO:teuthology.orchestra.run.smithi028.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-09-17T23:43:46.051 INFO:teuthology.orchestra.run.smithi049.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/main amd64 libreadline-dev amd64 8.1.2-1 [166 kB] 2024-09-17T23:43:46.055 INFO:teuthology.orchestra.run.smithi049.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblua5.3-dev amd64 5.3.6-1build1 [167 kB] 2024-09-17T23:43:46.056 INFO:teuthology.orchestra.run.smithi028.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/main amd64 libreadline-dev amd64 8.1.2-1 [166 kB] 2024-09-17T23:43:46.060 INFO:teuthology.orchestra.run.smithi049.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua5.1 amd64 5.1.5-8.1build4 [94.6 kB] 2024-09-17T23:43:46.060 INFO:teuthology.orchestra.run.smithi028.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblua5.3-dev amd64 5.3.6-1build1 [167 kB] 2024-09-17T23:43:46.063 INFO:teuthology.orchestra.run.smithi049.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-any all 27ubuntu1 [5,034 B] 2024-09-17T23:43:46.065 INFO:teuthology.orchestra.run.smithi028.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua5.1 amd64 5.1.5-8.1build4 [94.6 kB] 2024-09-17T23:43:46.069 INFO:teuthology.orchestra.run.smithi028.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-any all 27ubuntu1 [5,034 B] 2024-09-17T23:43:46.144 INFO:teuthology.orchestra.run.smithi049.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 zip amd64 3.0-12build2 [176 kB] 2024-09-17T23:43:46.148 INFO:teuthology.orchestra.run.smithi049.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 unzip amd64 6.0-26ubuntu3.2 [175 kB] 2024-09-17T23:43:46.149 INFO:teuthology.orchestra.run.smithi028.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 zip amd64 3.0-12build2 [176 kB] 2024-09-17T23:43:46.153 INFO:teuthology.orchestra.run.smithi049.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/universe amd64 luarocks all 3.8.0+dfsg1-1 [140 kB] 2024-09-17T23:43:46.154 INFO:teuthology.orchestra.run.smithi028.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 unzip amd64 6.0-26ubuntu3.2 [175 kB] 2024-09-17T23:43:46.156 INFO:teuthology.orchestra.run.smithi049.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-09-17T23:43:46.158 INFO:teuthology.orchestra.run.smithi049.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-09-17T23:43:46.158 INFO:teuthology.orchestra.run.smithi028.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/universe amd64 luarocks all 3.8.0+dfsg1-1 [140 kB] 2024-09-17T23:43:46.159 INFO:teuthology.orchestra.run.smithi049.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-09-17T23:43:46.161 INFO:teuthology.orchestra.run.smithi049.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-09-17T23:43:46.161 INFO:teuthology.orchestra.run.smithi049.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-09-17T23:43:46.162 INFO:teuthology.orchestra.run.smithi049.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-09-17T23:43:46.162 INFO:teuthology.orchestra.run.smithi028.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-09-17T23:43:46.163 INFO:teuthology.orchestra.run.smithi028.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-09-17T23:43:46.164 INFO:teuthology.orchestra.run.smithi028.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-09-17T23:43:46.166 INFO:teuthology.orchestra.run.smithi028.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-09-17T23:43:46.167 INFO:teuthology.orchestra.run.smithi028.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-09-17T23:43:46.167 INFO:teuthology.orchestra.run.smithi028.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-09-17T23:43:46.226 INFO:teuthology.orchestra.run.smithi049.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-09-17T23:43:46.232 INFO:teuthology.orchestra.run.smithi028.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-09-17T23:43:46.245 INFO:teuthology.orchestra.run.smithi049.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-09-17T23:43:46.245 INFO:teuthology.orchestra.run.smithi049.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-09-17T23:43:46.247 INFO:teuthology.orchestra.run.smithi049.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-09-17T23:43:46.250 INFO:teuthology.orchestra.run.smithi028.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-09-17T23:43:46.251 INFO:teuthology.orchestra.run.smithi028.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-09-17T23:43:46.252 INFO:teuthology.orchestra.run.smithi049.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-09-17T23:43:46.252 INFO:teuthology.orchestra.run.smithi028.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-09-17T23:43:46.252 INFO:teuthology.orchestra.run.smithi049.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-09-17T23:43:46.257 INFO:teuthology.orchestra.run.smithi028.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-09-17T23:43:46.258 INFO:teuthology.orchestra.run.smithi028.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-09-17T23:43:46.308 INFO:teuthology.orchestra.run.smithi049.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-09-17T23:43:46.309 INFO:teuthology.orchestra.run.smithi049.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-09-17T23:43:46.311 INFO:teuthology.orchestra.run.smithi049.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-09-17T23:43:46.311 INFO:teuthology.orchestra.run.smithi049.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-09-17T23:43:46.314 INFO:teuthology.orchestra.run.smithi028.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-09-17T23:43:46.314 INFO:teuthology.orchestra.run.smithi028.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-09-17T23:43:46.317 INFO:teuthology.orchestra.run.smithi028.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-09-17T23:43:46.317 INFO:teuthology.orchestra.run.smithi028.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-09-17T23:43:46.389 INFO:teuthology.orchestra.run.smithi049.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2024-09-17T23:43:46.391 INFO:teuthology.orchestra.run.smithi049.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-09-17T23:43:46.393 INFO:teuthology.orchestra.run.smithi049.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-09-17T23:43:46.394 INFO:teuthology.orchestra.run.smithi049.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-09-17T23:43:46.395 INFO:teuthology.orchestra.run.smithi028.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2024-09-17T23:43:46.397 INFO:teuthology.orchestra.run.smithi028.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-09-17T23:43:46.399 INFO:teuthology.orchestra.run.smithi028.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-09-17T23:43:46.399 INFO:teuthology.orchestra.run.smithi028.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-09-17T23:43:46.405 INFO:teuthology.orchestra.run.smithi049.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-09-17T23:43:46.410 INFO:teuthology.orchestra.run.smithi028.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-09-17T23:43:46.471 INFO:teuthology.orchestra.run.smithi049.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-09-17T23:43:46.472 INFO:teuthology.orchestra.run.smithi049.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-09-17T23:43:46.473 INFO:teuthology.orchestra.run.smithi049.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-09-17T23:43:46.475 INFO:teuthology.orchestra.run.smithi049.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-09-17T23:43:46.476 INFO:teuthology.orchestra.run.smithi028.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-09-17T23:43:46.477 INFO:teuthology.orchestra.run.smithi028.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-09-17T23:43:46.478 INFO:teuthology.orchestra.run.smithi028.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-09-17T23:43:46.479 INFO:teuthology.orchestra.run.smithi049.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-09-17T23:43:46.480 INFO:teuthology.orchestra.run.smithi028.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-09-17T23:43:46.484 INFO:teuthology.orchestra.run.smithi028.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-09-17T23:43:46.552 INFO:teuthology.orchestra.run.smithi049.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-09-17T23:43:46.554 INFO:teuthology.orchestra.run.smithi049.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-09-17T23:43:46.556 INFO:teuthology.orchestra.run.smithi049.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-09-17T23:43:46.557 INFO:teuthology.orchestra.run.smithi028.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-09-17T23:43:46.560 INFO:teuthology.orchestra.run.smithi028.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-09-17T23:43:46.561 INFO:teuthology.orchestra.run.smithi049.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-09-17T23:43:46.561 INFO:teuthology.orchestra.run.smithi028.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-09-17T23:43:46.562 INFO:teuthology.orchestra.run.smithi049.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-09-17T23:43:46.566 INFO:teuthology.orchestra.run.smithi028.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-09-17T23:43:46.567 INFO:teuthology.orchestra.run.smithi028.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-09-17T23:43:46.633 INFO:teuthology.orchestra.run.smithi049.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-asyncssh all 2.5.0-1 [185 kB] 2024-09-17T23:43:46.638 INFO:teuthology.orchestra.run.smithi049.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-09-17T23:43:46.639 INFO:teuthology.orchestra.run.smithi028.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-asyncssh all 2.5.0-1 [185 kB] 2024-09-17T23:43:46.644 INFO:teuthology.orchestra.run.smithi028.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-09-17T23:43:46.771 INFO:teuthology.orchestra.run.smithi028.stdout:Get:63 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 19.1.1-330-gc2ddf972-1jammy [5,145 kB] 2024-09-17T23:43:46.794 INFO:teuthology.orchestra.run.smithi049.stdout:Get:63 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 19.1.1-330-gc2ddf972-1jammy [5,145 kB] 2024-09-17T23:43:46.800 INFO:teuthology.orchestra.run.smithi049.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-09-17T23:43:46.802 INFO:teuthology.orchestra.run.smithi049.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-09-17T23:43:46.805 INFO:teuthology.orchestra.run.smithi049.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-09-17T23:43:46.805 INFO:teuthology.orchestra.run.smithi049.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-09-17T23:43:46.806 INFO:teuthology.orchestra.run.smithi028.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-09-17T23:43:46.808 INFO:teuthology.orchestra.run.smithi049.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-09-17T23:43:46.846 INFO:teuthology.orchestra.run.smithi049.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-09-17T23:43:46.849 INFO:teuthology.orchestra.run.smithi049.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-09-17T23:43:46.849 INFO:teuthology.orchestra.run.smithi049.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-09-17T23:43:46.859 INFO:teuthology.orchestra.run.smithi028.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-09-17T23:43:46.859 INFO:teuthology.orchestra.run.smithi028.stdout:Get:66 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 19.1.1-330-gc2ddf972-1jammy [245 kB] 2024-09-17T23:43:46.862 INFO:teuthology.orchestra.run.smithi028.stdout:Get:67 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 19.1.1-330-gc2ddf972-1jammy [124 kB] 2024-09-17T23:43:46.862 INFO:teuthology.orchestra.run.smithi028.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-09-17T23:43:46.863 INFO:teuthology.orchestra.run.smithi028.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-09-17T23:43:46.863 INFO:teuthology.orchestra.run.smithi028.stdout:Get:70 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 19.1.1-330-gc2ddf972-1jammy [1,555 kB] 2024-09-17T23:43:46.865 INFO:teuthology.orchestra.run.smithi028.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-09-17T23:43:46.871 INFO:teuthology.orchestra.run.smithi049.stdout:Get:72 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 19.1.1-330-gc2ddf972-1jammy [245 kB] 2024-09-17T23:43:46.879 INFO:teuthology.orchestra.run.smithi049.stdout:Get:73 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 19.1.1-330-gc2ddf972-1jammy [124 kB] 2024-09-17T23:43:46.881 INFO:teuthology.orchestra.run.smithi049.stdout:Get:74 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 19.1.1-330-gc2ddf972-1jammy [1,555 kB] 2024-09-17T23:43:46.894 INFO:teuthology.orchestra.run.smithi028.stdout:Get:72 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 19.1.1-330-gc2ddf972-1jammy [6,143 kB] 2024-09-17T23:43:46.905 INFO:teuthology.orchestra.run.smithi049.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-09-17T23:43:46.905 INFO:teuthology.orchestra.run.smithi049.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-09-17T23:43:46.910 INFO:teuthology.orchestra.run.smithi028.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-09-17T23:43:46.914 INFO:teuthology.orchestra.run.smithi028.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-09-17T23:43:46.914 INFO:teuthology.orchestra.run.smithi028.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-09-17T23:43:46.933 INFO:teuthology.orchestra.run.smithi049.stdout:Get:77 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 19.1.1-330-gc2ddf972-1jammy [6,143 kB] 2024-09-17T23:43:46.966 INFO:teuthology.orchestra.run.smithi049.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-09-17T23:43:46.967 INFO:teuthology.orchestra.run.smithi049.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-09-17T23:43:46.968 INFO:teuthology.orchestra.run.smithi049.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-09-17T23:43:46.970 INFO:teuthology.orchestra.run.smithi049.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-09-17T23:43:46.970 INFO:teuthology.orchestra.run.smithi049.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-09-17T23:43:46.971 INFO:teuthology.orchestra.run.smithi049.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-09-17T23:43:46.974 INFO:teuthology.orchestra.run.smithi028.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-09-17T23:43:46.974 INFO:teuthology.orchestra.run.smithi028.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-09-17T23:43:46.979 INFO:teuthology.orchestra.run.smithi049.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-09-17T23:43:47.025 INFO:teuthology.orchestra.run.smithi028.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-09-17T23:43:47.026 INFO:teuthology.orchestra.run.smithi028.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-09-17T23:43:47.026 INFO:teuthology.orchestra.run.smithi028.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-09-17T23:43:47.028 INFO:teuthology.orchestra.run.smithi028.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-09-17T23:43:47.029 INFO:teuthology.orchestra.run.smithi028.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-09-17T23:43:47.030 INFO:teuthology.orchestra.run.smithi028.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-09-17T23:43:47.038 INFO:teuthology.orchestra.run.smithi028.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-09-17T23:43:47.048 INFO:teuthology.orchestra.run.smithi049.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-09-17T23:43:47.051 INFO:teuthology.orchestra.run.smithi049.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-09-17T23:43:47.053 INFO:teuthology.orchestra.run.smithi049.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-09-17T23:43:47.073 INFO:teuthology.orchestra.run.smithi028.stdout:Get:85 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 19.1.1-330-gc2ddf972-1jammy [22.8 MB] 2024-09-17T23:43:47.106 INFO:teuthology.orchestra.run.smithi028.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-09-17T23:43:47.109 INFO:teuthology.orchestra.run.smithi028.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-09-17T23:43:47.111 INFO:teuthology.orchestra.run.smithi028.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-09-17T23:43:47.129 INFO:teuthology.orchestra.run.smithi049.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-09-17T23:43:47.135 INFO:teuthology.orchestra.run.smithi049.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-socket amd64 3.0~rc1+git+ac3201d-6 [78.9 kB] 2024-09-17T23:43:47.138 INFO:teuthology.orchestra.run.smithi049.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-sec amd64 1.0.2-1 [37.6 kB] 2024-09-17T23:43:47.139 INFO:teuthology.orchestra.run.smithi049.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2024-09-17T23:43:47.150 INFO:teuthology.orchestra.run.smithi049.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/main amd64 pkg-config amd64 0.29.2-1ubuntu3 [48.2 kB] 2024-09-17T23:43:47.152 INFO:teuthology.orchestra.run.smithi049.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python-asyncssh-doc all 2.5.0-1 [318 kB] 2024-09-17T23:43:47.159 INFO:teuthology.orchestra.run.smithi049.stdout:Get:94 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-09-17T23:43:47.173 INFO:teuthology.orchestra.run.smithi049.stdout:Get:95 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 19.1.1-330-gc2ddf972-1jammy [22.8 MB] 2024-09-17T23:43:47.188 INFO:teuthology.orchestra.run.smithi028.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-09-17T23:43:47.194 INFO:teuthology.orchestra.run.smithi028.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-socket amd64 3.0~rc1+git+ac3201d-6 [78.9 kB] 2024-09-17T23:43:47.196 INFO:teuthology.orchestra.run.smithi028.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-sec amd64 1.0.2-1 [37.6 kB] 2024-09-17T23:43:47.197 INFO:teuthology.orchestra.run.smithi028.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2024-09-17T23:43:47.209 INFO:teuthology.orchestra.run.smithi028.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy/main amd64 pkg-config amd64 0.29.2-1ubuntu3 [48.2 kB] 2024-09-17T23:43:47.210 INFO:teuthology.orchestra.run.smithi028.stdout:Get:94 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python-asyncssh-doc all 2.5.0-1 [318 kB] 2024-09-17T23:43:47.210 INFO:teuthology.orchestra.run.smithi049.stdout:Get:96 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-09-17T23:43:47.212 INFO:teuthology.orchestra.run.smithi049.stdout:Get:97 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-09-17T23:43:47.213 INFO:teuthology.orchestra.run.smithi049.stdout:Get:98 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-09-17T23:43:47.218 INFO:teuthology.orchestra.run.smithi028.stdout:Get:95 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-09-17T23:43:47.269 INFO:teuthology.orchestra.run.smithi028.stdout:Get:96 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-09-17T23:43:47.270 INFO:teuthology.orchestra.run.smithi028.stdout:Get:97 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-09-17T23:43:47.271 INFO:teuthology.orchestra.run.smithi028.stdout:Get:98 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-09-17T23:43:47.292 INFO:teuthology.orchestra.run.smithi049.stdout:Get:99 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-09-17T23:43:47.294 INFO:teuthology.orchestra.run.smithi049.stdout:Get:100 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-09-17T23:43:47.311 INFO:teuthology.orchestra.run.smithi049.stdout:Get:101 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-09-17T23:43:47.312 INFO:teuthology.orchestra.run.smithi049.stdout:Get:102 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-09-17T23:43:47.312 INFO:teuthology.orchestra.run.smithi049.stdout:Get:103 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-09-17T23:43:47.318 INFO:teuthology.orchestra.run.smithi049.stdout:Get:104 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-09-17T23:43:47.319 INFO:teuthology.orchestra.run.smithi049.stdout:Get:105 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-09-17T23:43:47.351 INFO:teuthology.orchestra.run.smithi028.stdout:Get:99 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-09-17T23:43:47.352 INFO:teuthology.orchestra.run.smithi028.stdout:Get:100 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-09-17T23:43:47.369 INFO:teuthology.orchestra.run.smithi028.stdout:Get:101 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-09-17T23:43:47.370 INFO:teuthology.orchestra.run.smithi028.stdout:Get:102 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-09-17T23:43:47.371 INFO:teuthology.orchestra.run.smithi028.stdout:Get:103 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-09-17T23:43:47.376 INFO:teuthology.orchestra.run.smithi028.stdout:Get:104 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-09-17T23:43:47.378 INFO:teuthology.orchestra.run.smithi028.stdout:Get:105 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-09-17T23:43:47.836 INFO:teuthology.orchestra.run.smithi028.stdout:Get:106 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 19.1.1-330-gc2ddf972-1jammy [14.1 kB] 2024-09-17T23:43:47.837 INFO:teuthology.orchestra.run.smithi028.stdout:Get:107 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 19.1.1-330-gc2ddf972-1jammy [877 kB] 2024-09-17T23:43:47.879 INFO:teuthology.orchestra.run.smithi028.stdout:Get:108 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 19.1.1-330-gc2ddf972-1jammy [2,462 kB] 2024-09-17T23:43:47.925 INFO:teuthology.orchestra.run.smithi028.stdout:Get:109 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 19.1.1-330-gc2ddf972-1jammy [794 kB] 2024-09-17T23:43:47.936 INFO:teuthology.orchestra.run.smithi028.stdout:Get:110 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 19.1.1-330-gc2ddf972-1jammy [152 kB] 2024-09-17T23:43:47.938 INFO:teuthology.orchestra.run.smithi028.stdout:Get:111 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 19.1.1-330-gc2ddf972-1jammy [2,275 kB] 2024-09-17T23:43:47.991 INFO:teuthology.orchestra.run.smithi028.stdout:Get:112 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 19.1.1-330-gc2ddf972-1jammy [8,625 kB] 2024-09-17T23:43:48.177 INFO:teuthology.orchestra.run.smithi028.stdout:Get:113 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 19.1.1-330-gc2ddf972-1jammy [14.2 kB] 2024-09-17T23:43:48.177 INFO:teuthology.orchestra.run.smithi028.stdout:Get:114 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 19.1.1-330-gc2ddf972-1jammy [51.4 MB] 2024-09-17T23:43:48.200 INFO:teuthology.orchestra.run.smithi049.stdout:Get:106 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 19.1.1-330-gc2ddf972-1jammy [14.1 kB] 2024-09-17T23:43:48.201 INFO:teuthology.orchestra.run.smithi049.stdout:Get:107 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 19.1.1-330-gc2ddf972-1jammy [877 kB] 2024-09-17T23:43:48.219 INFO:teuthology.orchestra.run.smithi049.stdout:Get:108 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 19.1.1-330-gc2ddf972-1jammy [2,462 kB] 2024-09-17T23:43:48.262 INFO:teuthology.orchestra.run.smithi049.stdout:Get:109 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 19.1.1-330-gc2ddf972-1jammy [794 kB] 2024-09-17T23:43:48.273 INFO:teuthology.orchestra.run.smithi049.stdout:Get:110 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 19.1.1-330-gc2ddf972-1jammy [152 kB] 2024-09-17T23:43:48.275 INFO:teuthology.orchestra.run.smithi049.stdout:Get:111 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 19.1.1-330-gc2ddf972-1jammy [2,275 kB] 2024-09-17T23:43:48.327 INFO:teuthology.orchestra.run.smithi049.stdout:Get:112 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 19.1.1-330-gc2ddf972-1jammy [8,625 kB] 2024-09-17T23:43:48.629 INFO:teuthology.orchestra.run.smithi049.stdout:Get:113 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 19.1.1-330-gc2ddf972-1jammy [14.2 kB] 2024-09-17T23:43:48.629 INFO:teuthology.orchestra.run.smithi049.stdout:Get:114 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 19.1.1-330-gc2ddf972-1jammy [51.4 MB] 2024-09-17T23:43:49.937 INFO:teuthology.orchestra.run.smithi028.stdout:Get:115 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 19.1.1-330-gc2ddf972-1jammy [130 kB] 2024-09-17T23:43:49.940 INFO:teuthology.orchestra.run.smithi028.stdout:Get:116 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 19.1.1-330-gc2ddf972-1jammy [39.0 kB] 2024-09-17T23:43:49.941 INFO:teuthology.orchestra.run.smithi028.stdout:Get:117 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 19.1.1-330-gc2ddf972-1jammy [13.6 MB] 2024-09-17T23:43:50.340 INFO:teuthology.orchestra.run.smithi028.stdout:Get:118 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 19.1.1-330-gc2ddf972-1jammy [91.4 kB] 2024-09-17T23:43:50.342 INFO:teuthology.orchestra.run.smithi028.stdout:Get:119 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 19.1.1-330-gc2ddf972-1jammy [3,140 kB] 2024-09-17T23:43:50.398 INFO:teuthology.orchestra.run.smithi028.stdout:Get:120 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 rbd-nbd amd64 19.1.1-330-gc2ddf972-1jammy [179 kB] 2024-09-17T23:43:50.500 INFO:teuthology.orchestra.run.smithi049.stdout:Get:115 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 19.1.1-330-gc2ddf972-1jammy [130 kB] 2024-09-17T23:43:50.503 INFO:teuthology.orchestra.run.smithi049.stdout:Get:116 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 19.1.1-330-gc2ddf972-1jammy [39.0 kB] 2024-09-17T23:43:50.504 INFO:teuthology.orchestra.run.smithi049.stdout:Get:117 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 19.1.1-330-gc2ddf972-1jammy [13.6 MB] 2024-09-17T23:43:50.696 INFO:teuthology.orchestra.run.smithi049.stdout:Get:118 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 19.1.1-330-gc2ddf972-1jammy [91.4 kB] 2024-09-17T23:43:50.697 INFO:teuthology.orchestra.run.smithi049.stdout:Get:119 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 19.1.1-330-gc2ddf972-1jammy [3,140 kB] 2024-09-17T23:43:50.747 INFO:teuthology.orchestra.run.smithi049.stdout:Get:120 https://3.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/ubuntu/jammy/flavors/default jammy/main amd64 rbd-nbd amd64 19.1.1-330-gc2ddf972-1jammy [179 kB] 2024-09-17T23:43:50.998 INFO:teuthology.orchestra.run.smithi028.stdout:Fetched 184 MB in 5s (34.5 MB/s) 2024-09-17T23:43:51.167 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-09-17T23:43:51.271 INFO:teuthology.orchestra.run.smithi028.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165990 files and directories currently installed.) 2024-09-17T23:43:51.277 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../000-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-09-17T23:43:51.351 INFO:teuthology.orchestra.run.smithi049.stdout:Fetched 184 MB in 6s (32.4 MB/s) 2024-09-17T23:43:51.357 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-17T23:43:51.485 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-09-17T23:43:51.586 INFO:teuthology.orchestra.run.smithi049.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165990 files and directories currently installed.) 2024-09-17T23:43:51.591 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../000-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-09-17T23:43:51.659 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-17T23:43:51.710 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-09-17T23:43:51.737 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../001-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-09-17T23:43:51.794 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-17T23:43:52.004 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-09-17T23:43:52.030 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../001-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-09-17T23:43:52.070 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-17T23:43:52.113 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-09-17T23:43:52.140 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../002-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-09-17T23:43:52.179 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-17T23:43:52.390 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-09-17T23:43:52.417 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../002-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-09-17T23:43:52.456 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-17T23:43:52.625 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-09-17T23:43:52.652 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../003-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-17T23:43:52.699 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-17T23:43:52.895 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-09-17T23:43:52.922 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../003-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-17T23:43:52.971 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-17T23:43:53.077 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-09-17T23:43:53.093 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../004-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-17T23:43:53.152 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-17T23:43:53.365 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-09-17T23:43:53.392 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../004-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-17T23:43:53.431 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-17T23:43:53.497 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-09-17T23:43:53.523 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../005-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-17T23:43:53.563 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-17T23:43:53.735 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-09-17T23:43:53.761 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../005-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-17T23:43:53.793 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-17T23:43:53.891 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-09-17T23:43:53.918 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../006-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-09-17T23:43:53.958 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-17T23:43:54.154 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-09-17T23:43:54.181 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../006-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-09-17T23:43:54.221 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-17T23:43:54.277 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package librados2. 2024-09-17T23:43:54.304 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../007-librados2_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:43:54.344 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking librados2 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:54.715 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package librados2. 2024-09-17T23:43:54.742 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../007-librados2_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:43:54.782 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking librados2 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:54.821 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package librbd1. 2024-09-17T23:43:54.848 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../008-librbd1_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:43:54.888 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking librbd1 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:55.218 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package librbd1. 2024-09-17T23:43:55.245 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../008-librbd1_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:43:55.285 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking librbd1 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:55.349 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libcephfs2. 2024-09-17T23:43:55.376 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../009-libcephfs2_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:43:55.416 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libcephfs2 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:55.688 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libcephfs2. 2024-09-17T23:43:55.701 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-rados. 2024-09-17T23:43:55.714 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../009-libcephfs2_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:43:55.718 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../010-python3-rados_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:43:55.754 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libcephfs2 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:55.759 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-rados (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:56.040 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-rados. 2024-09-17T23:43:56.056 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../010-python3-rados_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:43:56.062 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-09-17T23:43:56.089 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../011-python3-ceph-argparse_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:43:56.098 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-rados (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:56.128 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-ceph-argparse (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:56.339 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-cephfs. 2024-09-17T23:43:56.355 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../012-python3-cephfs_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:43:56.367 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-09-17T23:43:56.394 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../011-python3-ceph-argparse_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:43:56.397 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-cephfs (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:56.434 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-ceph-argparse (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:56.695 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-cephfs. 2024-09-17T23:43:56.717 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-ceph-common. 2024-09-17T23:43:56.722 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../012-python3-cephfs_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:43:56.744 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../013-python3-ceph-common_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:43:56.762 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-cephfs (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:56.784 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-ceph-common (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:57.073 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-ceph-common. 2024-09-17T23:43:57.099 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../013-python3-ceph-common_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:43:57.136 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-wcwidth. 2024-09-17T23:43:57.139 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-ceph-common (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:57.163 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../014-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-09-17T23:43:57.203 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-17T23:43:57.475 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-wcwidth. 2024-09-17T23:43:57.502 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../014-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-09-17T23:43:57.522 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-prettytable. 2024-09-17T23:43:57.542 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-17T23:43:57.549 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../015-python3-prettytable_2.5.0-2_all.deb ... 2024-09-17T23:43:57.589 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-09-17T23:43:57.803 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-prettytable. 2024-09-17T23:43:57.819 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../015-python3-prettytable_2.5.0-2_all.deb ... 2024-09-17T23:43:57.824 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-rbd. 2024-09-17T23:43:57.841 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../016-python3-rbd_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:43:57.853 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-09-17T23:43:57.874 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-rbd (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:58.106 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-rbd. 2024-09-17T23:43:58.121 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../016-python3-rbd_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:43:58.181 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-rbd (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:43:58.519 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-09-17T23:43:58.546 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../017-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-09-17T23:43:58.585 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-17T23:43:58.717 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-09-17T23:43:58.744 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../017-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-09-17T23:43:58.783 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-17T23:43:58.871 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libreadline-dev:amd64. 2024-09-17T23:43:58.898 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../018-libreadline-dev_8.1.2-1_amd64.deb ... 2024-09-17T23:43:58.938 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libreadline-dev:amd64 (8.1.2-1) ... 2024-09-17T23:43:59.061 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libreadline-dev:amd64. 2024-09-17T23:43:59.077 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../018-libreadline-dev_8.1.2-1_amd64.deb ... 2024-09-17T23:43:59.111 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libreadline-dev:amd64 (8.1.2-1) ... 2024-09-17T23:43:59.232 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package liblua5.3-dev:amd64. 2024-09-17T23:43:59.260 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../019-liblua5.3-dev_5.3.6-1build1_amd64.deb ... 2024-09-17T23:43:59.299 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking liblua5.3-dev:amd64 (5.3.6-1build1) ... 2024-09-17T23:43:59.389 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package liblua5.3-dev:amd64. 2024-09-17T23:43:59.405 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../019-liblua5.3-dev_5.3.6-1build1_amd64.deb ... 2024-09-17T23:43:59.438 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking liblua5.3-dev:amd64 (5.3.6-1build1) ... 2024-09-17T23:43:59.610 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package lua5.1. 2024-09-17T23:43:59.637 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../020-lua5.1_5.1.5-8.1build4_amd64.deb ... 2024-09-17T23:43:59.668 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking lua5.1 (5.1.5-8.1build4) ... 2024-09-17T23:43:59.775 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package lua5.1. 2024-09-17T23:43:59.802 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../020-lua5.1_5.1.5-8.1build4_amd64.deb ... 2024-09-17T23:43:59.842 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking lua5.1 (5.1.5-8.1build4) ... 2024-09-17T23:43:59.988 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package lua-any. 2024-09-17T23:44:00.004 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../021-lua-any_27ubuntu1_all.deb ... 2024-09-17T23:44:00.046 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking lua-any (27ubuntu1) ... 2024-09-17T23:44:00.153 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package lua-any. 2024-09-17T23:44:00.180 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../021-lua-any_27ubuntu1_all.deb ... 2024-09-17T23:44:00.220 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking lua-any (27ubuntu1) ... 2024-09-17T23:44:00.298 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package zip. 2024-09-17T23:44:00.325 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../022-zip_3.0-12build2_amd64.deb ... 2024-09-17T23:44:00.365 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking zip (3.0-12build2) ... 2024-09-17T23:44:00.439 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package zip. 2024-09-17T23:44:00.455 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../022-zip_3.0-12build2_amd64.deb ... 2024-09-17T23:44:00.489 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking zip (3.0-12build2) ... 2024-09-17T23:44:00.610 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package unzip. 2024-09-17T23:44:00.626 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../023-unzip_6.0-26ubuntu3.2_amd64.deb ... 2024-09-17T23:44:00.668 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking unzip (6.0-26ubuntu3.2) ... 2024-09-17T23:44:00.825 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package unzip. 2024-09-17T23:44:00.852 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../023-unzip_6.0-26ubuntu3.2_amd64.deb ... 2024-09-17T23:44:00.892 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking unzip (6.0-26ubuntu3.2) ... 2024-09-17T23:44:01.088 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package luarocks. 2024-09-17T23:44:01.115 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../024-luarocks_3.8.0+dfsg1-1_all.deb ... 2024-09-17T23:44:01.155 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking luarocks (3.8.0+dfsg1-1) ... 2024-09-17T23:44:01.287 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package luarocks. 2024-09-17T23:44:01.314 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../024-luarocks_3.8.0+dfsg1-1_all.deb ... 2024-09-17T23:44:01.354 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking luarocks (3.8.0+dfsg1-1) ... 2024-09-17T23:44:01.529 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package librgw2. 2024-09-17T23:44:01.545 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../025-librgw2_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:01.578 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking librgw2 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:01.745 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package librgw2. 2024-09-17T23:44:01.772 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../025-librgw2_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:01.812 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking librgw2 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:02.106 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-rgw. 2024-09-17T23:44:02.122 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../026-python3-rgw_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:02.164 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-rgw (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:02.381 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-rgw. 2024-09-17T23:44:02.407 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../026-python3-rgw_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:02.439 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-rgw (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:02.450 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package liboath0:amd64. 2024-09-17T23:44:02.466 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../027-liboath0_2.6.7-3build1_amd64.deb ... 2024-09-17T23:44:02.500 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-09-17T23:44:02.776 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package liboath0:amd64. 2024-09-17T23:44:02.802 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../027-liboath0_2.6.7-3build1_amd64.deb ... 2024-09-17T23:44:02.812 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libradosstriper1. 2024-09-17T23:44:02.839 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../028-libradosstriper1_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:02.841 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-09-17T23:44:02.878 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libradosstriper1 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:03.144 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libradosstriper1. 2024-09-17T23:44:03.172 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../028-libradosstriper1_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:03.211 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libradosstriper1 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:03.457 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-common. 2024-09-17T23:44:03.484 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../029-ceph-common_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:03.523 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-common (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:03.573 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-common. 2024-09-17T23:44:03.600 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../029-ceph-common_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:03.656 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-common (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:04.953 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-base. 2024-09-17T23:44:04.980 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../030-ceph-base_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:05.029 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-base (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:05.070 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-base. 2024-09-17T23:44:05.097 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../030-ceph-base_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:05.146 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-base (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:05.574 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-09-17T23:44:05.601 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../031-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-09-17T23:44:05.640 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-09-17T23:44:05.724 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-09-17T23:44:05.751 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../031-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-09-17T23:44:05.790 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-09-17T23:44:05.918 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-cheroot. 2024-09-17T23:44:05.945 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../032-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-09-17T23:44:05.976 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-17T23:44:06.118 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-cheroot. 2024-09-17T23:44:06.145 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../032-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-09-17T23:44:06.185 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-17T23:44:06.237 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-09-17T23:44:06.264 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../033-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-09-17T23:44:06.295 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-09-17T23:44:06.521 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-09-17T23:44:06.548 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../033-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-09-17T23:44:06.573 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-jaraco.text. 2024-09-17T23:44:06.587 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-09-17T23:44:06.600 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../034-python3-jaraco.text_3.6.0-2_all.deb ... 2024-09-17T23:44:06.640 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-09-17T23:44:06.865 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-jaraco.text. 2024-09-17T23:44:06.892 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../034-python3-jaraco.text_3.6.0-2_all.deb ... 2024-09-17T23:44:06.924 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-09-17T23:44:06.942 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-09-17T23:44:06.970 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../035-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-09-17T23:44:07.009 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-09-17T23:44:07.193 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-09-17T23:44:07.220 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../035-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-09-17T23:44:07.268 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-09-17T23:44:07.278 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-tempora. 2024-09-17T23:44:07.306 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../036-python3-tempora_4.1.2-1_all.deb ... 2024-09-17T23:44:07.337 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-09-17T23:44:07.571 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-tempora. 2024-09-17T23:44:07.598 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../036-python3-tempora_4.1.2-1_all.deb ... 2024-09-17T23:44:07.606 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-portend. 2024-09-17T23:44:07.633 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../037-python3-portend_3.0.0-1_all.deb ... 2024-09-17T23:44:07.637 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-09-17T23:44:07.673 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-09-17T23:44:07.940 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-portend. 2024-09-17T23:44:07.967 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../037-python3-portend_3.0.0-1_all.deb ... 2024-09-17T23:44:07.975 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-09-17T23:44:08.003 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../038-python3-zc.lockfile_2.0-1_all.deb ... 2024-09-17T23:44:08.007 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-09-17T23:44:08.042 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-09-17T23:44:08.268 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-09-17T23:44:08.295 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../038-python3-zc.lockfile_2.0-1_all.deb ... 2024-09-17T23:44:08.326 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-09-17T23:44:08.345 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-cherrypy3. 2024-09-17T23:44:08.372 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../039-python3-cherrypy3_18.6.1-4_all.deb ... 2024-09-17T23:44:08.411 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-09-17T23:44:08.629 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-cherrypy3. 2024-09-17T23:44:08.656 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../039-python3-cherrypy3_18.6.1-4_all.deb ... 2024-09-17T23:44:08.695 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-09-17T23:44:09.015 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-natsort. 2024-09-17T23:44:09.042 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../040-python3-natsort_8.0.2-1_all.deb ... 2024-09-17T23:44:09.081 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-09-17T23:44:09.208 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-natsort. 2024-09-17T23:44:09.235 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../040-python3-natsort_8.0.2-1_all.deb ... 2024-09-17T23:44:09.274 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-09-17T23:44:09.351 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-logutils. 2024-09-17T23:44:09.378 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../041-python3-logutils_0.3.3-8_all.deb ... 2024-09-17T23:44:09.409 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-09-17T23:44:09.561 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-logutils. 2024-09-17T23:44:09.588 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../041-python3-logutils_0.3.3-8_all.deb ... 2024-09-17T23:44:09.627 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-09-17T23:44:09.687 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-markupsafe. 2024-09-17T23:44:09.714 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../042-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-09-17T23:44:09.753 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-09-17T23:44:09.946 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-markupsafe. 2024-09-17T23:44:09.974 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../042-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-09-17T23:44:10.013 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-09-17T23:44:10.056 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-mako. 2024-09-17T23:44:10.079 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../043-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-09-17T23:44:10.122 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-17T23:44:10.316 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-mako. 2024-09-17T23:44:10.343 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../043-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-09-17T23:44:10.382 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-17T23:44:10.451 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-simplegeneric. 2024-09-17T23:44:10.478 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../044-python3-simplegeneric_0.8.1-3_all.deb ... 2024-09-17T23:44:10.517 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-09-17T23:44:10.711 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-simplegeneric. 2024-09-17T23:44:10.738 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../044-python3-simplegeneric_0.8.1-3_all.deb ... 2024-09-17T23:44:10.777 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-09-17T23:44:10.820 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-singledispatch. 2024-09-17T23:44:10.846 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../045-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-09-17T23:44:10.886 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-09-17T23:44:11.072 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-singledispatch. 2024-09-17T23:44:11.099 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../045-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-09-17T23:44:11.138 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-09-17T23:44:11.189 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-webob. 2024-09-17T23:44:11.216 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../046-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2024-09-17T23:44:11.255 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-17T23:44:11.399 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-webob. 2024-09-17T23:44:11.423 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../046-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2024-09-17T23:44:11.457 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-17T23:44:11.575 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-waitress. 2024-09-17T23:44:11.603 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../047-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-09-17T23:44:11.637 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-17T23:44:11.785 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-waitress. 2024-09-17T23:44:11.812 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../047-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-09-17T23:44:11.863 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-17T23:44:11.886 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-tempita. 2024-09-17T23:44:11.913 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../048-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-09-17T23:44:11.953 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-09-17T23:44:12.171 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-tempita. 2024-09-17T23:44:12.199 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../048-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-09-17T23:44:12.238 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-09-17T23:44:12.239 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-paste. 2024-09-17T23:44:12.266 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../049-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-09-17T23:44:12.305 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-09-17T23:44:12.524 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-paste. 2024-09-17T23:44:12.551 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../049-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-09-17T23:44:12.590 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-09-17T23:44:12.600 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-09-17T23:44:12.616 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../050-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-09-17T23:44:12.650 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-09-17T23:44:12.911 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-09-17T23:44:12.928 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-pastedeploy. 2024-09-17T23:44:12.938 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../050-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-09-17T23:44:12.954 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../051-python3-pastedeploy_2.1.1-1_all.deb ... 2024-09-17T23:44:12.977 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-09-17T23:44:12.994 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-09-17T23:44:13.297 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-webtest. 2024-09-17T23:44:13.313 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-pastedeploy. 2024-09-17T23:44:13.324 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../052-python3-webtest_2.0.35-1_all.deb ... 2024-09-17T23:44:13.340 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../051-python3-pastedeploy_2.1.1-1_all.deb ... 2024-09-17T23:44:13.363 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-09-17T23:44:13.380 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-09-17T23:44:13.675 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-pecan. 2024-09-17T23:44:13.691 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-webtest. 2024-09-17T23:44:13.701 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../053-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-09-17T23:44:13.718 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../052-python3-webtest_2.0.35-1_all.deb ... 2024-09-17T23:44:13.741 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-09-17T23:44:13.757 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-09-17T23:44:14.036 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-pecan. 2024-09-17T23:44:14.063 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../053-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-09-17T23:44:14.094 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-09-17T23:44:14.104 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-certifi. 2024-09-17T23:44:14.131 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../054-python3-certifi_2020.6.20-1_all.deb ... 2024-09-17T23:44:14.170 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-09-17T23:44:14.415 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-certifi. 2024-09-17T23:44:14.432 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../054-python3-certifi_2020.6.20-1_all.deb ... 2024-09-17T23:44:14.448 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-idna. 2024-09-17T23:44:14.466 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-09-17T23:44:14.466 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../055-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-09-17T23:44:14.498 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-09-17T23:44:14.793 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-idna. 2024-09-17T23:44:14.809 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-urllib3. 2024-09-17T23:44:14.820 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../055-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-09-17T23:44:14.836 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../056-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-09-17T23:44:14.859 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-09-17T23:44:14.876 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-17T23:44:15.179 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-urllib3. 2024-09-17T23:44:15.187 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-requests. 2024-09-17T23:44:15.206 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../056-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-09-17T23:44:15.214 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../057-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-09-17T23:44:15.245 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-17T23:44:15.253 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-17T23:44:15.565 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-werkzeug. 2024-09-17T23:44:15.592 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../058-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-09-17T23:44:15.631 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-17T23:44:15.640 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-requests. 2024-09-17T23:44:15.667 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../057-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-09-17T23:44:15.707 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-17T23:44:15.918 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-09-17T23:44:15.934 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../059-ceph-mgr-modules-core_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:44:15.976 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-mgr-modules-core (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:15.976 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-werkzeug. 2024-09-17T23:44:16.003 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../058-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-09-17T23:44:16.034 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-17T23:44:16.337 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-09-17T23:44:16.338 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-09-17T23:44:16.353 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../059-ceph-mgr-modules-core_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:44:16.355 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../060-libsqlite3-mod-ceph_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:16.388 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libsqlite3-mod-ceph (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:16.396 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-mgr-modules-core (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:16.667 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-bcrypt. 2024-09-17T23:44:16.693 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../061-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-09-17T23:44:16.733 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-09-17T23:44:16.776 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-09-17T23:44:16.803 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../060-libsqlite3-mod-ceph_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:16.842 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libsqlite3-mod-ceph (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:17.019 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-openssl. 2024-09-17T23:44:17.046 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../062-python3-openssl_21.0.0-1_all.deb ... 2024-09-17T23:44:17.085 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-09-17T23:44:17.137 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-bcrypt. 2024-09-17T23:44:17.164 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../061-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-09-17T23:44:17.203 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-09-17T23:44:17.438 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-mgr. 2024-09-17T23:44:17.447 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-openssl. 2024-09-17T23:44:17.464 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../062-python3-openssl_21.0.0-1_all.deb ... 2024-09-17T23:44:17.465 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../063-ceph-mgr_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:17.497 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-09-17T23:44:17.505 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-mgr (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:17.825 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-mgr. 2024-09-17T23:44:17.852 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../063-ceph-mgr_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:17.866 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-mon. 2024-09-17T23:44:17.891 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../064-ceph-mon_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:17.891 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-mgr (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:17.924 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-mon (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:18.303 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-mon. 2024-09-17T23:44:18.330 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../064-ceph-mon_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:18.370 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-mon (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:19.678 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-osd. 2024-09-17T23:44:19.705 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../065-ceph-osd_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:19.744 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-osd (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:19.841 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-osd. 2024-09-17T23:44:19.867 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../065-ceph-osd_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:19.898 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-osd (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:20.956 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph. 2024-09-17T23:44:20.972 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../066-ceph_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:20.976 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph. 2024-09-17T23:44:20.993 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../066-ceph_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:21.014 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:21.034 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:21.258 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-fuse. 2024-09-17T23:44:21.286 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../067-ceph-fuse_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:21.317 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-fuse (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:21.379 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-fuse. 2024-09-17T23:44:21.406 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../067-ceph-fuse_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:21.446 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-fuse (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:21.686 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-mds. 2024-09-17T23:44:21.713 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../068-ceph-mds_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:21.753 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-mds (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:21.799 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-mds. 2024-09-17T23:44:21.826 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../068-ceph-mds_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:21.857 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-mds (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:22.172 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package cephadm. 2024-09-17T23:44:22.199 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../069-cephadm_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:22.235 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package cephadm. 2024-09-17T23:44:22.238 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking cephadm (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:22.262 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../069-cephadm_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:22.301 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking cephadm (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:22.574 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-asyncssh. 2024-09-17T23:44:22.601 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../070-python3-asyncssh_2.5.0-1_all.deb ... 2024-09-17T23:44:22.604 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-asyncssh. 2024-09-17T23:44:22.631 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../070-python3-asyncssh_2.5.0-1_all.deb ... 2024-09-17T23:44:22.641 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-asyncssh (2.5.0-1) ... 2024-09-17T23:44:22.670 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-asyncssh (2.5.0-1) ... 2024-09-17T23:44:22.960 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python-babel-localedata. 2024-09-17T23:44:22.976 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../071-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-09-17T23:44:23.007 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python-babel-localedata. 2024-09-17T23:44:23.018 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-17T23:44:23.034 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../071-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-09-17T23:44:23.073 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-17T23:44:24.020 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-babel. 2024-09-17T23:44:24.040 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-babel. 2024-09-17T23:44:24.047 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../072-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-09-17T23:44:24.068 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../072-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-09-17T23:44:24.086 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-09-17T23:44:24.124 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-09-17T23:44:24.399 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-jinja2. 2024-09-17T23:44:24.422 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../073-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-09-17T23:44:24.434 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-jinja2. 2024-09-17T23:44:24.462 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../073-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-09-17T23:44:24.464 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-17T23:44:24.501 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-17T23:44:24.808 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-09-17T23:44:24.825 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../074-ceph-mgr-cephadm_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:44:24.828 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-09-17T23:44:24.856 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../074-ceph-mgr-cephadm_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:44:24.858 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-mgr-cephadm (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:24.887 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-mgr-cephadm (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:25.179 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-repoze.lru. 2024-09-17T23:44:25.206 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../075-python3-repoze.lru_0.7-2_all.deb ... 2024-09-17T23:44:25.206 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-repoze.lru. 2024-09-17T23:44:25.234 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../075-python3-repoze.lru_0.7-2_all.deb ... 2024-09-17T23:44:25.236 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-09-17T23:44:25.273 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-09-17T23:44:25.534 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-routes. 2024-09-17T23:44:25.540 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-routes. 2024-09-17T23:44:25.562 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../076-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-09-17T23:44:25.567 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../076-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-09-17T23:44:25.601 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-09-17T23:44:25.606 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-09-17T23:44:25.895 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-09-17T23:44:25.923 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../077-ceph-mgr-dashboard_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:44:25.926 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-09-17T23:44:25.953 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../077-ceph-mgr-dashboard_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:44:25.953 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-mgr-dashboard (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:25.992 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-mgr-dashboard (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:27.200 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-09-17T23:44:27.216 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../078-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-09-17T23:44:27.230 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-09-17T23:44:27.247 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../078-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-09-17T23:44:27.249 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-17T23:44:27.289 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-17T23:44:27.714 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-joblib. 2024-09-17T23:44:27.730 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../079-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-09-17T23:44:27.764 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-09-17T23:44:27.767 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-joblib. 2024-09-17T23:44:27.795 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../079-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-09-17T23:44:27.833 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-09-17T23:44:28.150 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-09-17T23:44:28.178 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../080-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-09-17T23:44:28.196 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-09-17T23:44:28.217 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-09-17T23:44:28.223 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../080-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-09-17T23:44:28.262 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-09-17T23:44:28.537 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-sklearn. 2024-09-17T23:44:28.548 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-sklearn. 2024-09-17T23:44:28.564 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../081-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-09-17T23:44:28.576 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../081-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-09-17T23:44:28.603 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-17T23:44:28.614 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-17T23:44:29.295 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-09-17T23:44:29.323 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-09-17T23:44:29.323 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../082-ceph-mgr-diskprediction-local_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:44:29.351 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../082-ceph-mgr-diskprediction-local_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:44:29.447 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-mgr-diskprediction-local (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:29.475 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-mgr-diskprediction-local (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:30.429 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-cachetools. 2024-09-17T23:44:30.456 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../083-python3-cachetools_5.0.0-1_all.deb ... 2024-09-17T23:44:30.495 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-09-17T23:44:30.617 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-cachetools. 2024-09-17T23:44:30.645 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../083-python3-cachetools_5.0.0-1_all.deb ... 2024-09-17T23:44:30.683 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-09-17T23:44:30.773 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-pyasn1. 2024-09-17T23:44:30.800 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../084-python3-pyasn1_0.4.8-1_all.deb ... 2024-09-17T23:44:30.831 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-09-17T23:44:30.961 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-pyasn1. 2024-09-17T23:44:30.989 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../084-python3-pyasn1_0.4.8-1_all.deb ... 2024-09-17T23:44:31.020 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-09-17T23:44:31.117 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-09-17T23:44:31.134 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../085-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-09-17T23:44:31.167 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-09-17T23:44:31.323 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-09-17T23:44:31.351 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../085-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-09-17T23:44:31.389 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-09-17T23:44:31.464 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-rsa. 2024-09-17T23:44:31.492 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../086-python3-rsa_4.8-1_all.deb ... 2024-09-17T23:44:31.530 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-rsa (4.8-1) ... 2024-09-17T23:44:31.717 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-rsa. 2024-09-17T23:44:31.745 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../086-python3-rsa_4.8-1_all.deb ... 2024-09-17T23:44:31.802 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-rsa (4.8-1) ... 2024-09-17T23:44:31.852 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-google-auth. 2024-09-17T23:44:31.879 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../087-python3-google-auth_1.5.1-3_all.deb ... 2024-09-17T23:44:31.917 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-09-17T23:44:32.121 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-google-auth. 2024-09-17T23:44:32.149 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../087-python3-google-auth_1.5.1-3_all.deb ... 2024-09-17T23:44:32.187 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-09-17T23:44:32.246 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-09-17T23:44:32.273 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../088-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-09-17T23:44:32.312 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-17T23:44:32.482 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-09-17T23:44:32.510 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../088-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-09-17T23:44:32.549 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-17T23:44:32.624 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-websocket. 2024-09-17T23:44:32.652 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../089-python3-websocket_1.2.3-1_all.deb ... 2024-09-17T23:44:32.690 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-09-17T23:44:32.860 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-websocket. 2024-09-17T23:44:32.888 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../089-python3-websocket_1.2.3-1_all.deb ... 2024-09-17T23:44:32.927 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-09-17T23:44:33.019 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-kubernetes. 2024-09-17T23:44:33.047 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../090-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-09-17T23:44:33.077 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-17T23:44:33.247 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-kubernetes. 2024-09-17T23:44:33.275 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../090-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-09-17T23:44:33.313 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-17T23:44:33.749 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-09-17T23:44:33.778 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../091-ceph-mgr-k8sevents_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:44:33.816 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-mgr-k8sevents (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:33.977 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-09-17T23:44:33.994 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../091-ceph-mgr-k8sevents_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:44:34.035 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-mgr-k8sevents (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:34.219 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libonig5:amd64. 2024-09-17T23:44:34.246 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../092-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-09-17T23:44:34.302 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-17T23:44:34.363 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libonig5:amd64. 2024-09-17T23:44:34.391 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../092-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-09-17T23:44:34.430 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-17T23:44:34.647 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libjq1:amd64. 2024-09-17T23:44:34.675 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../093-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-09-17T23:44:34.713 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-17T23:44:34.783 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libjq1:amd64. 2024-09-17T23:44:34.811 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../093-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-09-17T23:44:34.841 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-17T23:44:34.974 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package jq. 2024-09-17T23:44:34.991 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../094-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-09-17T23:44:35.094 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package jq. 2024-09-17T23:44:35.110 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../094-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-09-17T23:44:35.147 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-09-17T23:44:35.152 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-09-17T23:44:35.397 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package socat. 2024-09-17T23:44:35.413 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../095-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-09-17T23:44:35.455 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-09-17T23:44:35.510 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package socat. 2024-09-17T23:44:35.538 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../095-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-09-17T23:44:35.577 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-09-17T23:44:35.792 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package xmlstarlet. 2024-09-17T23:44:35.819 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../096-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-09-17T23:44:35.858 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-09-17T23:44:35.880 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package xmlstarlet. 2024-09-17T23:44:35.908 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../096-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-09-17T23:44:35.947 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-09-17T23:44:36.221 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-test. 2024-09-17T23:44:36.237 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../097-ceph-test_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:36.279 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-test (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:36.301 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-test. 2024-09-17T23:44:36.317 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../097-ceph-test_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:36.359 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-test (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:38.608 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package ceph-volume. 2024-09-17T23:44:38.624 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../098-ceph-volume_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:44:38.658 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking ceph-volume (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:38.756 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package ceph-volume. 2024-09-17T23:44:38.783 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../098-ceph-volume_19.1.1-330-gc2ddf972-1jammy_all.deb ... 2024-09-17T23:44:38.822 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking ceph-volume (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:38.961 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package libcephfs-dev. 2024-09-17T23:44:38.978 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../099-libcephfs-dev_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:39.012 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking libcephfs-dev (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:39.118 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package libcephfs-dev. 2024-09-17T23:44:39.146 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../099-libcephfs-dev_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:39.184 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking libcephfs-dev (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:39.314 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package lua-socket:amd64. 2024-09-17T23:44:39.342 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../100-lua-socket_3.0~rc1+git+ac3201d-6_amd64.deb ... 2024-09-17T23:44:39.381 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2024-09-17T23:44:39.504 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package lua-socket:amd64. 2024-09-17T23:44:39.531 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../100-lua-socket_3.0~rc1+git+ac3201d-6_amd64.deb ... 2024-09-17T23:44:39.570 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2024-09-17T23:44:39.744 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package lua-sec:amd64. 2024-09-17T23:44:39.772 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../101-lua-sec_1.0.2-1_amd64.deb ... 2024-09-17T23:44:39.810 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking lua-sec:amd64 (1.0.2-1) ... 2024-09-17T23:44:39.892 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package lua-sec:amd64. 2024-09-17T23:44:39.920 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../101-lua-sec_1.0.2-1_amd64.deb ... 2024-09-17T23:44:39.958 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking lua-sec:amd64 (1.0.2-1) ... 2024-09-17T23:44:40.173 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package nvme-cli. 2024-09-17T23:44:40.201 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../102-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2024-09-17T23:44:40.239 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2024-09-17T23:44:40.449 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package nvme-cli. 2024-09-17T23:44:40.477 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../102-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2024-09-17T23:44:40.520 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2024-09-17T23:44:40.844 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package pkg-config. 2024-09-17T23:44:40.873 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../103-pkg-config_0.29.2-1ubuntu3_amd64.deb ... 2024-09-17T23:44:40.909 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package pkg-config. 2024-09-17T23:44:40.910 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking pkg-config (0.29.2-1ubuntu3) ... 2024-09-17T23:44:40.937 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../103-pkg-config_0.29.2-1ubuntu3_amd64.deb ... 2024-09-17T23:44:40.975 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking pkg-config (0.29.2-1ubuntu3) ... 2024-09-17T23:44:41.164 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python-asyncssh-doc. 2024-09-17T23:44:41.180 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../104-python-asyncssh-doc_2.5.0-1_all.deb ... 2024-09-17T23:44:41.222 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python-asyncssh-doc (2.5.0-1) ... 2024-09-17T23:44:41.237 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python-asyncssh-doc. 2024-09-17T23:44:41.253 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../104-python-asyncssh-doc_2.5.0-1_all.deb ... 2024-09-17T23:44:41.295 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python-asyncssh-doc (2.5.0-1) ... 2024-09-17T23:44:41.633 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-iniconfig. 2024-09-17T23:44:41.661 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../105-python3-iniconfig_1.1.1-2_all.deb ... 2024-09-17T23:44:41.692 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-09-17T23:44:41.723 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-iniconfig. 2024-09-17T23:44:41.751 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../105-python3-iniconfig_1.1.1-2_all.deb ... 2024-09-17T23:44:41.789 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-09-17T23:44:41.994 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-pastescript. 2024-09-17T23:44:42.022 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../106-python3-pastescript_2.0.2-4_all.deb ... 2024-09-17T23:44:42.061 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-09-17T23:44:42.076 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-pastescript. 2024-09-17T23:44:42.104 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../106-python3-pastescript_2.0.2-4_all.deb ... 2024-09-17T23:44:42.142 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-09-17T23:44:42.381 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-pluggy. 2024-09-17T23:44:42.409 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../107-python3-pluggy_0.13.0-7.1_all.deb ... 2024-09-17T23:44:42.448 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-09-17T23:44:42.454 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-pluggy. 2024-09-17T23:44:42.482 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../107-python3-pluggy_0.13.0-7.1_all.deb ... 2024-09-17T23:44:42.520 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-09-17T23:44:42.767 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-psutil. 2024-09-17T23:44:42.795 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../108-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-09-17T23:44:42.823 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-psutil. 2024-09-17T23:44:42.833 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-09-17T23:44:42.851 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../108-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-09-17T23:44:42.890 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-09-17T23:44:43.170 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-py. 2024-09-17T23:44:43.198 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../109-python3-py_1.10.0-1_all.deb ... 2024-09-17T23:44:43.243 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-py. 2024-09-17T23:44:43.244 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-py (1.10.0-1) ... 2024-09-17T23:44:43.271 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../109-python3-py_1.10.0-1_all.deb ... 2024-09-17T23:44:43.318 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-py (1.10.0-1) ... 2024-09-17T23:44:43.598 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-pygments. 2024-09-17T23:44:43.621 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-pygments. 2024-09-17T23:44:43.626 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../110-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-09-17T23:44:43.649 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../110-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-09-17T23:44:43.664 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-09-17T23:44:43.687 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-09-17T23:44:44.077 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-pyinotify. 2024-09-17T23:44:44.105 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../111-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-09-17T23:44:44.142 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-pyinotify. 2024-09-17T23:44:44.143 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-09-17T23:44:44.170 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../111-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-09-17T23:44:44.208 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-09-17T23:44:44.455 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-toml. 2024-09-17T23:44:44.483 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../112-python3-toml_0.10.2-1_all.deb ... 2024-09-17T23:44:44.521 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-09-17T23:44:44.553 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-toml. 2024-09-17T23:44:44.581 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../112-python3-toml_0.10.2-1_all.deb ... 2024-09-17T23:44:44.619 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-09-17T23:44:44.824 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-pytest. 2024-09-17T23:44:44.852 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../113-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-09-17T23:44:44.890 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-09-17T23:44:44.931 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-pytest. 2024-09-17T23:44:44.959 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../113-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-09-17T23:44:44.997 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-09-17T23:44:45.244 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package python3-simplejson. 2024-09-17T23:44:45.272 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../114-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-09-17T23:44:45.310 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-09-17T23:44:45.351 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package python3-simplejson. 2024-09-17T23:44:45.379 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../114-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-09-17T23:44:45.417 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-09-17T23:44:45.588 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-09-17T23:44:45.605 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../115-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-09-17T23:44:45.646 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-09-17T23:44:45.669 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-09-17T23:44:45.687 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../115-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-09-17T23:44:45.728 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-09-17T23:44:46.334 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package radosgw. 2024-09-17T23:44:46.351 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../116-radosgw_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:46.384 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking radosgw (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:46.399 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package radosgw. 2024-09-17T23:44:46.427 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../116-radosgw_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:46.457 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking radosgw (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:47.253 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package rbd-fuse. 2024-09-17T23:44:47.279 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../117-rbd-fuse_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:47.285 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package rbd-fuse. 2024-09-17T23:44:47.302 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../117-rbd-fuse_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:47.311 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking rbd-fuse (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:47.343 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking rbd-fuse (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:47.640 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package rbd-mirror. 2024-09-17T23:44:47.668 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../118-rbd-mirror_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:47.680 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package rbd-mirror. 2024-09-17T23:44:47.706 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking rbd-mirror (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:47.707 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../118-rbd-mirror_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:47.746 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking rbd-mirror (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:48.182 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package rbd-nbd. 2024-09-17T23:44:48.199 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../119-rbd-nbd_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:48.240 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking rbd-nbd (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:48.250 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package rbd-nbd. 2024-09-17T23:44:48.267 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../119-rbd-nbd_19.1.1-330-gc2ddf972-1jammy_amd64.deb ... 2024-09-17T23:44:48.308 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking rbd-nbd (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:48.521 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-09-17T23:44:48.597 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-09-17T23:44:48.779 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-17T23:44:48.855 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-17T23:44:48.905 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2024-09-17T23:44:48.981 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2024-09-17T23:44:49.172 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-09-17T23:44:49.248 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-09-17T23:44:49.529 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-09-17T23:44:49.555 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-09-17T23:44:49.953 INFO:teuthology.orchestra.run.smithi049.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-09-17T23:44:49.978 INFO:teuthology.orchestra.run.smithi028.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-09-17T23:44:50.023 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up cephadm (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:50.050 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up cephadm (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:50.474 INFO:teuthology.orchestra.run.smithi049.stdout:Adding system user cephadm....done 2024-09-17T23:44:50.631 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-17T23:44:50.818 INFO:teuthology.orchestra.run.smithi028.stdout:Adding system user cephadm....done 2024-09-17T23:44:50.965 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-17T23:44:51.181 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-09-17T23:44:51.325 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-09-17T23:44:51.416 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python-asyncssh-doc (2.5.0-1) ... 2024-09-17T23:44:51.533 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-09-17T23:44:51.568 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python-asyncssh-doc (2.5.0-1) ... 2024-09-17T23:44:51.669 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-09-17T23:44:51.768 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-09-17T23:44:51.912 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-09-17T23:44:52.052 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-09-17T23:44:52.178 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-py (1.10.0-1) ... 2024-09-17T23:44:52.179 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-09-17T23:44:52.297 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-py (1.10.0-1) ... 2024-09-17T23:44:52.497 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-09-17T23:44:52.624 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-09-17T23:44:52.858 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-09-17T23:44:52.984 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-09-17T23:44:53.092 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-09-17T23:44:53.235 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-09-17T23:44:53.343 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up unzip (6.0-26ubuntu3.2) ... 2024-09-17T23:44:53.512 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up unzip (6.0-26ubuntu3.2) ... 2024-09-17T23:44:53.595 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-09-17T23:44:53.772 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-09-17T23:44:53.854 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-09-17T23:44:54.031 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-09-17T23:44:54.122 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-09-17T23:44:54.307 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-09-17T23:44:54.381 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-ceph-argparse (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:54.558 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-ceph-argparse (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:44:54.640 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-17T23:44:54.758 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2024-09-17T23:44:54.847 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-17T23:44:54.859 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libreadline-dev:amd64 (8.1.2-1) ... 2024-09-17T23:44:54.960 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-09-17T23:44:54.968 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2024-09-17T23:44:55.103 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libreadline-dev:amd64 (8.1.2-1) ... 2024-09-17T23:44:55.212 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up lua5.1 (5.1.5-8.1build4) ... 2024-09-17T23:44:55.229 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-09-17T23:44:55.497 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up lua5.1 (5.1.5-8.1build4) ... 2024-09-17T23:44:55.585 INFO:teuthology.orchestra.run.smithi049.stdout:update-alternatives: using /usr/bin/lua5.1 to provide /usr/bin/lua (lua-interpreter) in auto mode 2024-09-17T23:44:55.618 INFO:teuthology.orchestra.run.smithi049.stdout:update-alternatives: using /usr/bin/luac5.1 to provide /usr/bin/luac (lua-compiler) in auto mode 2024-09-17T23:44:55.673 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-17T23:44:55.807 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-09-17T23:44:55.862 INFO:teuthology.orchestra.run.smithi028.stdout:update-alternatives: using /usr/bin/lua5.1 to provide /usr/bin/lua (lua-interpreter) in auto mode 2024-09-17T23:44:55.886 INFO:teuthology.orchestra.run.smithi028.stdout:update-alternatives: using /usr/bin/luac5.1 to provide /usr/bin/luac (lua-compiler) in auto mode 2024-09-17T23:44:55.958 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-17T23:44:56.068 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-09-17T23:44:56.167 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-17T23:44:56.268 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-09-17T23:44:56.527 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-09-17T23:44:56.635 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-17T23:44:56.761 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-09-17T23:44:56.803 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-09-17T23:44:57.038 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-09-17T23:44:57.138 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up zip (3.0-12build2) ... 2024-09-17T23:44:57.264 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-09-17T23:44:57.305 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-09-17T23:44:57.598 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up zip (3.0-12build2) ... 2024-09-17T23:44:57.724 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-09-17T23:44:57.950 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-09-17T23:44:58.376 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-09-17T23:44:58.426 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-09-17T23:44:58.502 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-09-17T23:44:58.628 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-17T23:44:58.703 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-09-17T23:44:58.820 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-09-17T23:44:58.937 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-asyncssh (2.5.0-1) ... 2024-09-17T23:44:58.946 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-17T23:44:59.247 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-asyncssh (2.5.0-1) ... 2024-09-17T23:44:59.339 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-09-17T23:44:59.616 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-09-17T23:44:59.623 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-09-17T23:44:59.874 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-09-17T23:45:00.016 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-17T23:45:00.251 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-17T23:45:00.318 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-17T23:45:00.536 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-17T23:45:00.686 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-09-17T23:45:00.920 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-09-17T23:45:00.937 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-09-17T23:45:01.054 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-ceph-common (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:01.180 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-09-17T23:45:01.308 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-ceph-common (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:01.348 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-09-17T23:45:01.599 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-09-17T23:45:01.674 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-17T23:45:01.925 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-17T23:45:02.888 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up pkg-config (0.29.2-1ubuntu3) ... 2024-09-17T23:45:03.115 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-17T23:45:03.164 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up pkg-config (0.29.2-1ubuntu3) ... 2024-09-17T23:45:03.249 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-toml (0.10.2-1) ... 2024-09-17T23:45:03.391 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-17T23:45:03.517 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-17T23:45:03.525 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-toml (0.10.2-1) ... 2024-09-17T23:45:03.643 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-09-17T23:45:03.769 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-17T23:45:03.793 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-17T23:45:03.894 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-09-17T23:45:04.003 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-17T23:45:04.079 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-09-17T23:45:04.313 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-09-17T23:45:04.330 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-09-17T23:45:04.564 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-17T23:45:04.572 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-09-17T23:45:04.690 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-09-17T23:45:04.823 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-17T23:45:04.949 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-09-17T23:45:05.017 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-09-17T23:45:05.251 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-09-17T23:45:05.276 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-09-17T23:45:05.585 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-09-17T23:45:05.619 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-09-17T23:45:05.861 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-09-17T23:45:05.878 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-09-17T23:45:06.104 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-09-17T23:45:06.130 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-09-17T23:45:06.381 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-17T23:45:06.397 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-09-17T23:45:06.498 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-09-17T23:45:06.673 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-17T23:45:06.799 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-09-17T23:45:06.800 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-17T23:45:06.926 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-17T23:45:07.084 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-17T23:45:07.185 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-17T23:45:07.227 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-17T23:45:07.461 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-17T23:45:07.545 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-09-17T23:45:07.770 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-09-17T23:45:07.796 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up liblua5.3-dev:amd64 (5.3.6-1build1) ... 2024-09-17T23:45:07.905 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up lua-sec:amd64 (1.0.2-1) ... 2024-09-17T23:45:08.006 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-09-17T23:45:08.038 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up liblua5.3-dev:amd64 (5.3.6-1build1) ... 2024-09-17T23:45:08.097 INFO:teuthology.orchestra.run.smithi049.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-09-17T23:45:08.164 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up lua-sec:amd64 (1.0.2-1) ... 2024-09-17T23:45:08.302 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-09-17T23:45:08.336 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-17T23:45:08.379 INFO:teuthology.orchestra.run.smithi028.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-09-17T23:45:08.459 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-09-17T23:45:08.642 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-17T23:45:08.768 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-09-17T23:45:08.852 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-09-17T23:45:09.111 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up lua-any (27ubuntu1) ... 2024-09-17T23:45:09.177 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-09-17T23:45:09.221 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-portend (3.0.0-1) ... 2024-09-17T23:45:09.437 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up lua-any (27ubuntu1) ... 2024-09-17T23:45:09.472 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-17T23:45:09.563 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-portend (3.0.0-1) ... 2024-09-17T23:45:09.581 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-17T23:45:09.822 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-17T23:45:09.924 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-09-17T23:45:09.948 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-17T23:45:10.225 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-17T23:45:10.300 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-09-17T23:45:10.518 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-09-17T23:45:10.609 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-17T23:45:10.636 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-rsa (4.8-1) ... 2024-09-17T23:45:10.860 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-09-17T23:45:10.904 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-09-17T23:45:10.961 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-rsa (4.8-1) ... 2024-09-17T23:45:11.172 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-17T23:45:11.220 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-09-17T23:45:11.488 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-17T23:45:11.516 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-09-17T23:45:11.867 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-09-17T23:45:11.902 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-09-17T23:45:12.170 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-09-17T23:45:12.244 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-09-17T23:45:12.514 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-17T23:45:12.520 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-09-17T23:45:12.640 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up librados2 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:12.766 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libsqlite3-mod-ceph (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:12.883 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-17T23:45:12.892 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up luarocks (3.8.0+dfsg1-1) ... 2024-09-17T23:45:13.001 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up librados2 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:13.102 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libsqlite3-mod-ceph (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:13.187 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libcephfs2 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:13.203 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up luarocks (3.8.0+dfsg1-1) ... 2024-09-17T23:45:13.313 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libradosstriper1 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:13.456 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-09-17T23:45:13.539 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libcephfs2 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:13.673 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libradosstriper1 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:13.732 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up librbd1 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:13.799 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-09-17T23:45:13.858 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-mgr-modules-core (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:13.984 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-fuse (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:14.076 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up librbd1 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:14.185 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-mgr-modules-core (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:14.204 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-17T23:45:14.204 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-17T23:45:14.287 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-fuse (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:14.493 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-17T23:45:14.493 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-17T23:45:14.661 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up libcephfs-dev (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:14.787 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-rados (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:14.922 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up librgw2 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:14.987 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up libcephfs-dev (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:15.065 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-17T23:45:15.114 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-rados (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:15.240 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up librgw2 (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:15.366 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-17T23:45:16.357 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-rbd (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:16.466 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up rbd-fuse (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:16.609 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up rbd-nbd (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:16.636 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-rbd (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:16.735 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-rgw (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:16.737 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up rbd-fuse (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:16.861 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up python3-cephfs (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:16.863 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up rbd-nbd (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:16.987 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-common (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:16.989 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-rgw (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:17.132 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up python3-cephfs (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:17.258 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-common (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:17.480 INFO:teuthology.orchestra.run.smithi049.stdout:Adding group ceph....done 2024-09-17T23:45:17.767 INFO:teuthology.orchestra.run.smithi028.stdout:Adding group ceph....done 2024-09-17T23:45:17.857 INFO:teuthology.orchestra.run.smithi049.stdout:Adding system user ceph....done 2024-09-17T23:45:18.025 INFO:teuthology.orchestra.run.smithi049.stdout:Setting system user ceph properties....done 2024-09-17T23:45:18.041 INFO:teuthology.orchestra.run.smithi049.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-09-17T23:45:18.178 INFO:teuthology.orchestra.run.smithi028.stdout:Adding system user ceph....done 2024-09-17T23:45:18.182 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-09-17T23:45:18.312 INFO:teuthology.orchestra.run.smithi028.stdout:Setting system user ceph properties....done 2024-09-17T23:45:18.326 INFO:teuthology.orchestra.run.smithi028.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-09-17T23:45:18.476 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-09-17T23:45:18.485 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-09-17T23:45:18.787 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-09-17T23:45:18.972 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-test (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:19.107 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up radosgw (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:19.251 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-test (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:19.376 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up radosgw (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:19.620 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-17T23:45:19.621 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-17T23:45:19.814 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-17T23:45:19.814 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-17T23:45:20.042 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-base (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:20.245 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-base (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:20.404 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-09-17T23:45:20.588 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-09-17T23:45:20.877 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-mds (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:21.013 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-mds (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:21.114 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-17T23:45:21.114 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-17T23:45:21.230 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-17T23:45:21.231 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-17T23:45:21.587 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up rbd-mirror (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:21.749 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up rbd-mirror (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:21.967 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target -> /lib/systemd/system/ceph-rbd-mirror.target. 2024-09-17T23:45:21.967 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target -> /lib/systemd/system/ceph-rbd-mirror.target. 2024-09-17T23:45:22.130 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target -> /lib/systemd/system/ceph-rbd-mirror.target. 2024-09-17T23:45:22.130 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target -> /lib/systemd/system/ceph-rbd-mirror.target. 2024-09-17T23:45:22.433 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-mgr (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:22.630 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-mgr (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:22.686 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-17T23:45:22.687 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-17T23:45:22.876 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-17T23:45:22.876 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-17T23:45:23.143 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-osd (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:23.357 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-osd (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:23.399 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-17T23:45:23.400 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-17T23:45:23.607 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-17T23:45:23.608 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-17T23:45:23.861 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-mgr-k8sevents (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:23.979 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-mgr-diskprediction-local (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:24.092 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-mgr-k8sevents (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:24.147 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-mon (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:24.218 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-mgr-diskprediction-local (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:24.357 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-17T23:45:24.357 INFO:teuthology.orchestra.run.smithi028.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-17T23:45:24.369 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-mon (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:24.588 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-17T23:45:24.588 INFO:teuthology.orchestra.run.smithi049.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-17T23:45:24.851 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-mgr-cephadm (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:25.045 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-mgr-cephadm (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:25.061 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:25.188 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:25.263 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-mgr-dashboard (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:25.314 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-mgr-dashboard (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:25.465 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up ceph-volume (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:25.516 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up ceph-volume (19.1.1-330-gc2ddf972-1jammy) ... 2024-09-17T23:45:25.835 INFO:teuthology.orchestra.run.smithi049.stdout:Processing triggers for mailcap (3.70+nmu1ubuntu1) ... 2024-09-17T23:45:25.944 INFO:teuthology.orchestra.run.smithi049.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-09-17T23:45:25.955 INFO:teuthology.orchestra.run.smithi028.stdout:Processing triggers for mailcap (3.70+nmu1ubuntu1) ... 2024-09-17T23:45:26.056 INFO:teuthology.orchestra.run.smithi028.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-09-17T23:45:26.096 INFO:teuthology.orchestra.run.smithi049.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-09-17T23:45:26.209 INFO:teuthology.orchestra.run.smithi028.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-09-17T23:45:26.838 INFO:teuthology.orchestra.run.smithi049.stdout:Processing triggers for install-info (6.8-4build1) ... 2024-09-17T23:45:27.067 INFO:teuthology.orchestra.run.smithi028.stdout:Processing triggers for install-info (6.8-4build1) ... 2024-09-17T23:45:27.566 INFO:teuthology.orchestra.run.smithi049.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-17T23:45:27.570 DEBUG:teuthology.orchestra.run.smithi049:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install pv 2024-09-17T23:45:27.671 INFO:teuthology.orchestra.run.smithi049.stdout:Reading package lists... 2024-09-17T23:45:27.778 INFO:teuthology.orchestra.run.smithi028.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-17T23:45:27.781 DEBUG:teuthology.orchestra.run.smithi028:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install pv 2024-09-17T23:45:27.879 INFO:teuthology.orchestra.run.smithi028.stdout:Reading package lists... 2024-09-17T23:45:27.887 INFO:teuthology.orchestra.run.smithi049.stdout:Building dependency tree... 2024-09-17T23:45:27.888 INFO:teuthology.orchestra.run.smithi049.stdout:Reading state information... 2024-09-17T23:45:28.094 INFO:teuthology.orchestra.run.smithi028.stdout:Building dependency tree... 2024-09-17T23:45:28.095 INFO:teuthology.orchestra.run.smithi028.stdout:Reading state information... 2024-09-17T23:45:28.098 INFO:teuthology.orchestra.run.smithi049.stdout:The following packages were automatically installed and are no longer required: 2024-09-17T23:45:28.098 INFO:teuthology.orchestra.run.smithi049.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-17T23:45:28.098 INFO:teuthology.orchestra.run.smithi049.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-17T23:45:28.098 INFO:teuthology.orchestra.run.smithi049.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-17T23:45:28.099 INFO:teuthology.orchestra.run.smithi049.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-17T23:45:28.099 INFO:teuthology.orchestra.run.smithi049.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-17T23:45:28.099 INFO:teuthology.orchestra.run.smithi049.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-17T23:45:28.101 INFO:teuthology.orchestra.run.smithi049.stdout:Suggested packages: 2024-09-17T23:45:28.101 INFO:teuthology.orchestra.run.smithi049.stdout: doc-base 2024-09-17T23:45:28.129 INFO:teuthology.orchestra.run.smithi049.stdout:The following NEW packages will be installed: 2024-09-17T23:45:28.129 INFO:teuthology.orchestra.run.smithi049.stdout: pv 2024-09-17T23:45:28.308 INFO:teuthology.orchestra.run.smithi028.stdout:The following packages were automatically installed and are no longer required: 2024-09-17T23:45:28.308 INFO:teuthology.orchestra.run.smithi028.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-17T23:45:28.308 INFO:teuthology.orchestra.run.smithi028.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-17T23:45:28.308 INFO:teuthology.orchestra.run.smithi028.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-17T23:45:28.309 INFO:teuthology.orchestra.run.smithi028.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-17T23:45:28.309 INFO:teuthology.orchestra.run.smithi028.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-17T23:45:28.309 INFO:teuthology.orchestra.run.smithi028.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-17T23:45:28.311 INFO:teuthology.orchestra.run.smithi028.stdout:Suggested packages: 2024-09-17T23:45:28.311 INFO:teuthology.orchestra.run.smithi028.stdout: doc-base 2024-09-17T23:45:28.313 INFO:teuthology.orchestra.run.smithi049.stdout:0 upgraded, 1 newly installed, 0 to remove and 331 not upgraded. 2024-09-17T23:45:28.313 INFO:teuthology.orchestra.run.smithi049.stdout:Need to get 44.5 kB of archives. 2024-09-17T23:45:28.313 INFO:teuthology.orchestra.run.smithi049.stdout:After this operation, 139 kB of additional disk space will be used. 2024-09-17T23:45:28.313 INFO:teuthology.orchestra.run.smithi049.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 pv amd64 1.6.6-1build2 [44.5 kB] 2024-09-17T23:45:28.339 INFO:teuthology.orchestra.run.smithi028.stdout:The following NEW packages will be installed: 2024-09-17T23:45:28.339 INFO:teuthology.orchestra.run.smithi028.stdout: pv 2024-09-17T23:45:28.394 INFO:teuthology.orchestra.run.smithi028.stdout:0 upgraded, 1 newly installed, 0 to remove and 331 not upgraded. 2024-09-17T23:45:28.394 INFO:teuthology.orchestra.run.smithi028.stdout:Need to get 44.5 kB of archives. 2024-09-17T23:45:28.395 INFO:teuthology.orchestra.run.smithi028.stdout:After this operation, 139 kB of additional disk space will be used. 2024-09-17T23:45:28.395 INFO:teuthology.orchestra.run.smithi028.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 pv amd64 1.6.6-1build2 [44.5 kB] 2024-09-17T23:45:28.776 INFO:teuthology.orchestra.run.smithi028.stdout:Fetched 44.5 kB in 0s (553 kB/s) 2024-09-17T23:45:28.835 INFO:teuthology.orchestra.run.smithi049.stdout:Fetched 44.5 kB in 0s (131 kB/s) 2024-09-17T23:45:28.863 INFO:teuthology.orchestra.run.smithi028.stdout:Selecting previously unselected package pv. 2024-09-17T23:45:28.910 INFO:teuthology.orchestra.run.smithi049.stdout:Selecting previously unselected package pv. 2024-09-17T23:45:28.923 INFO:teuthology.orchestra.run.smithi028.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 173908 files and directories currently installed.) 2024-09-17T23:45:28.928 INFO:teuthology.orchestra.run.smithi028.stdout:Preparing to unpack .../pv_1.6.6-1build2_amd64.deb ... 2024-09-17T23:45:28.972 INFO:teuthology.orchestra.run.smithi028.stdout:Unpacking pv (1.6.6-1build2) ... 2024-09-17T23:45:28.994 INFO:teuthology.orchestra.run.smithi049.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 173908 files and directories currently installed.) 2024-09-17T23:45:29.000 INFO:teuthology.orchestra.run.smithi049.stdout:Preparing to unpack .../pv_1.6.6-1build2_amd64.deb ... 2024-09-17T23:45:29.035 INFO:teuthology.orchestra.run.smithi049.stdout:Unpacking pv (1.6.6-1build2) ... 2024-09-17T23:45:29.299 INFO:teuthology.orchestra.run.smithi028.stdout:Setting up pv (1.6.6-1build2) ... 2024-09-17T23:45:29.379 INFO:teuthology.orchestra.run.smithi049.stdout:Setting up pv (1.6.6-1build2) ... 2024-09-17T23:45:29.435 INFO:teuthology.orchestra.run.smithi028.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-09-17T23:45:29.506 INFO:teuthology.orchestra.run.smithi049.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-09-17T23:45:30.867 INFO:teuthology.orchestra.run.smithi028.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-17T23:45:30.873 DEBUG:teuthology.parallel:result is None 2024-09-17T23:45:30.939 INFO:teuthology.orchestra.run.smithi049.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-17T23:45:30.943 DEBUG:teuthology.parallel:result is None 2024-09-17T23:45:30.944 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T23:45:31.083 DEBUG:teuthology.orchestra.run.smithi028:> dpkg-query -W -f '${Version}' ceph 2024-09-17T23:45:31.102 INFO:teuthology.orchestra.run.smithi028.stdout:19.1.1-330-gc2ddf972-1jammy 2024-09-17T23:45:31.103 INFO:teuthology.packaging:The installed version of ceph is 19.1.1-330-gc2ddf972-1jammy 2024-09-17T23:45:31.103 INFO:teuthology.task.install:The correct ceph version 19.1.1-330-gc2ddf972-1jammy is installed. 2024-09-17T23:45:31.105 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T23:45:31.241 DEBUG:teuthology.orchestra.run.smithi049:> dpkg-query -W -f '${Version}' ceph 2024-09-17T23:45:31.262 INFO:teuthology.orchestra.run.smithi049.stdout:19.1.1-330-gc2ddf972-1jammy 2024-09-17T23:45:31.262 INFO:teuthology.packaging:The installed version of ceph is 19.1.1-330-gc2ddf972-1jammy 2024-09-17T23:45:31.262 INFO:teuthology.task.install:The correct ceph version 19.1.1-330-gc2ddf972-1jammy is installed. 2024-09-17T23:45:31.265 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-17T23:45:31.265 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:31.266 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-17T23:45:31.281 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:31.281 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-17T23:45:31.319 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-17T23:45:31.319 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:31.319 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/usr/bin/daemon-helper 2024-09-17T23:45:31.336 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-17T23:45:31.392 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:31.392 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd of=/usr/bin/daemon-helper 2024-09-17T23:45:31.406 DEBUG:teuthology.orchestra.run.smithi049:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-17T23:45:31.461 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-17T23:45:31.461 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:31.461 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-17T23:45:31.473 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-17T23:45:31.528 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:31.528 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-17T23:45:31.542 DEBUG:teuthology.orchestra.run.smithi049:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-17T23:45:31.597 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-17T23:45:31.597 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:31.598 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/usr/bin/stdin-killer 2024-09-17T23:45:31.611 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-17T23:45:31.667 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:31.668 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd of=/usr/bin/stdin-killer 2024-09-17T23:45:31.681 DEBUG:teuthology.orchestra.run.smithi049:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-17T23:45:31.737 INFO:teuthology.run_tasks:Running task ceph... 2024-09-17T23:45:31.824 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2024-09-17T23:45:31.824 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 777 /var/log/ceph 2024-09-17T23:45:31.826 DEBUG:teuthology.orchestra.run.smithi049:> sudo chmod 777 /var/log/ceph 2024-09-17T23:45:31.840 INFO:tasks.ceph:Disabling ceph logrotate... 2024-09-17T23:45:31.840 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -f -- /etc/logrotate.d/ceph 2024-09-17T23:45:31.882 DEBUG:teuthology.orchestra.run.smithi049:> sudo rm -f -- /etc/logrotate.d/ceph 2024-09-17T23:45:31.896 INFO:tasks.ceph:Creating extra log directories... 2024-09-17T23:45:31.897 DEBUG:teuthology.orchestra.run.smithi028:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-09-17T23:45:31.937 DEBUG:teuthology.orchestra.run.smithi049:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-09-17T23:45:31.965 INFO:tasks.ceph:Creating ceph cluster cluster1... 2024-09-17T23:45:31.965 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-09-17T23:45:31.966 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601', 'branch': 'squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid supported-random-distro$/{ubuntu_latest} workloads/compare-mirror-images-nbd}', 'email': 'ceph-qa@ceph.com', 'first_in_suite': False, 'job_id': '7909601', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}], 'name': 'teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'extra_packages': ['rbd-nbd'], 'flavor': 'default', 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}, 'extra_system_packages': ['pv']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'squid', 'sha1': 'b0d356b98e08f6bc59ef15652f0d0e4298fc83c7'}}, 'owner': 'scheduled_teuthology@teuthology', 'priority': 100, 'repo': 'https://git.ceph.com/ceph.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 1983, 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f', 'sleep_before_teardown': 0, 'subset': '79/128', 'suite': 'rbd', 'suite_branch': 'squid', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph_b0d356b98e08f6bc59ef15652f0d0e4298fc83c7/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph.git', 'suite_sha1': 'b0d356b98e08f6bc59ef15652f0d0e4298fc83c7', 'targets': {'smithi028.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJ6R7D/dpH5Q/kJUJlgC8va/DNhJaP+PmCcJJPaHFjdILYt4sac7Kdqus1jDVzYR5Dp/oPvfM3MhfFS26p5pLlo=', 'smithi049.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKHINc9zC/b1hH/hNMeqnZWqar4lztbQ1kZBO25TL4fCb59Evgo2aYxq63Ywrhh+do4yrTzxrlD+89bwhBYec8o='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror', 'rbd-nbd'], 'flavor': 'default', 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f', 'extra_system_packages': ['pv']}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/compare_mirror_images.sh']}, 'env': {'RBD_DEVICE_TYPE': 'nbd'}, 'timeout': '3h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-09-17_21:16:02', 'tube': 'smithi', 'user': 'teuthology', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856'} 2024-09-17T23:45:31.966 DEBUG:teuthology.orchestra.run.smithi028:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2024-09-17T23:45:31.993 DEBUG:teuthology.orchestra.run.smithi049:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2024-09-17T23:45:32.002 DEBUG:teuthology.orchestra.run.smithi028:> sudo install -d -m0777 -- /var/run/ceph 2024-09-17T23:45:32.041 DEBUG:teuthology.orchestra.run.smithi049:> sudo install -d -m0777 -- /var/run/ceph 2024-09-17T23:45:32.058 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:32.058 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T23:45:32.100 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T23:45:32.101 DEBUG:teuthology.orchestra.run.smithi028:> stat /dev/vg_nvme/lv_1 2024-09-17T23:45:32.149 INFO:teuthology.orchestra.run.smithi028.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T23:45:32.149 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T23:45:32.149 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 5h/5d Inode: 887 Links: 1 2024-09-17T23:45:32.149 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T23:45:32.149 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-09-17 23:39:58.050660197 +0000 2024-09-17T23:45:32.149 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-09-17 23:39:58.046660233 +0000 2024-09-17T23:45:32.149 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-09-17 23:39:58.046660233 +0000 2024-09-17T23:45:32.149 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: - 2024-09-17T23:45:32.150 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T23:45:32.203 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records in 2024-09-17T23:45:32.203 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records out 2024-09-17T23:45:32.203 INFO:teuthology.orchestra.run.smithi028.stderr:512 bytes copied, 0.000246982 s, 2.1 MB/s 2024-09-17T23:45:32.205 DEBUG:teuthology.orchestra.run.smithi028:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T23:45:32.255 DEBUG:teuthology.orchestra.run.smithi028:> stat /dev/vg_nvme/lv_2 2024-09-17T23:45:32.305 INFO:teuthology.orchestra.run.smithi028.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T23:45:32.305 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T23:45:32.305 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 5h/5d Inode: 898 Links: 1 2024-09-17T23:45:32.306 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T23:45:32.306 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-09-17 23:39:58.794653552 +0000 2024-09-17T23:45:32.306 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-09-17 23:39:58.546655767 +0000 2024-09-17T23:45:32.306 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-09-17 23:39:58.546655767 +0000 2024-09-17T23:45:32.306 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: - 2024-09-17T23:45:32.306 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T23:45:32.359 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records in 2024-09-17T23:45:32.359 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records out 2024-09-17T23:45:32.359 INFO:teuthology.orchestra.run.smithi028.stderr:512 bytes copied, 0.000308081 s, 1.7 MB/s 2024-09-17T23:45:32.361 DEBUG:teuthology.orchestra.run.smithi028:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T23:45:32.411 DEBUG:teuthology.orchestra.run.smithi028:> stat /dev/vg_nvme/lv_3 2024-09-17T23:45:32.457 INFO:teuthology.orchestra.run.smithi028.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T23:45:32.457 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T23:45:32.457 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 5h/5d Inode: 915 Links: 1 2024-09-17T23:45:32.457 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T23:45:32.457 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-09-17 23:39:59.290649121 +0000 2024-09-17T23:45:32.458 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-09-17 23:39:59.030651444 +0000 2024-09-17T23:45:32.458 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-09-17 23:39:59.030651444 +0000 2024-09-17T23:45:32.458 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: - 2024-09-17T23:45:32.458 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T23:45:32.515 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records in 2024-09-17T23:45:32.515 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records out 2024-09-17T23:45:32.515 INFO:teuthology.orchestra.run.smithi028.stderr:512 bytes copied, 0.000254441 s, 2.0 MB/s 2024-09-17T23:45:32.517 DEBUG:teuthology.orchestra.run.smithi028:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T23:45:32.565 DEBUG:teuthology.orchestra.run.smithi028:> stat /dev/vg_nvme/lv_4 2024-09-17T23:45:32.612 INFO:teuthology.orchestra.run.smithi028.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T23:45:32.613 INFO:teuthology.orchestra.run.smithi028.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T23:45:32.613 INFO:teuthology.orchestra.run.smithi028.stdout:Device: 5h/5d Inode: 930 Links: 1 2024-09-17T23:45:32.613 INFO:teuthology.orchestra.run.smithi028.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T23:45:32.613 INFO:teuthology.orchestra.run.smithi028.stdout:Access: 2024-09-17 23:39:59.526647013 +0000 2024-09-17T23:45:32.613 INFO:teuthology.orchestra.run.smithi028.stdout:Modify: 2024-09-17 23:39:59.522647049 +0000 2024-09-17T23:45:32.613 INFO:teuthology.orchestra.run.smithi028.stdout:Change: 2024-09-17 23:39:59.522647049 +0000 2024-09-17T23:45:32.613 INFO:teuthology.orchestra.run.smithi028.stdout: Birth: - 2024-09-17T23:45:32.613 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T23:45:32.667 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records in 2024-09-17T23:45:32.667 INFO:teuthology.orchestra.run.smithi028.stderr:1+0 records out 2024-09-17T23:45:32.667 INFO:teuthology.orchestra.run.smithi028.stderr:512 bytes copied, 0.000304692 s, 1.7 MB/s 2024-09-17T23:45:32.668 DEBUG:teuthology.orchestra.run.smithi028:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T23:45:32.718 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-17T23:45:32.719 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi028.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-17T23:45:32.719 INFO:tasks.ceph:Generating config... 2024-09-17T23:45:32.720 INFO:tasks.ceph:[client] debug rbd = 20 2024-09-17T23:45:32.720 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2024-09-17T23:45:32.720 INFO:tasks.ceph:[client] log to stderr = False 2024-09-17T23:45:32.720 INFO:tasks.ceph:[client] rbd default features = 125 2024-09-17T23:45:32.720 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:32.720 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:32.720 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:32.720 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:32.721 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:32.721 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:32.721 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:32.721 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:32.721 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:32.721 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:32.721 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:32.721 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:32.721 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:32.721 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:32.721 INFO:tasks.ceph:[global] mon client directed command retry = 5 2024-09-17T23:45:32.721 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-09-17T23:45:32.721 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2024-09-17T23:45:32.722 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-09-17T23:45:32.722 INFO:tasks.ceph:[mgr] debug ms = 1 2024-09-17T23:45:32.722 INFO:tasks.ceph:[mon] debug mon = 20 2024-09-17T23:45:32.722 INFO:tasks.ceph:[mon] debug ms = 1 2024-09-17T23:45:32.722 INFO:tasks.ceph:[mon] debug paxos = 20 2024-09-17T23:45:32.722 INFO:tasks.ceph:[osd] bluefs allocator = hybrid 2024-09-17T23:45:32.722 INFO:tasks.ceph:[osd] bluestore allocator = hybrid 2024-09-17T23:45:32.722 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-09-17T23:45:32.722 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-09-17T23:45:32.722 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-09-17T23:45:32.722 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-09-17T23:45:32.722 INFO:tasks.ceph:[osd] debug ms = 1 2024-09-17T23:45:32.722 INFO:tasks.ceph:[osd] debug osd = 20 2024-09-17T23:45:32.722 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-09-17T23:45:32.722 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-09-17T23:45:32.723 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-09-17T23:45:32.723 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-09-17T23:45:32.723 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-09-17T23:45:32.723 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-09-17T23:45:32.723 INFO:tasks.ceph:Setting up cluster1.mon.a... 2024-09-17T23:45:32.723 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2024-09-17T23:45:32.792 INFO:teuthology.orchestra.run.smithi028.stdout:creating /etc/ceph/cluster1.keyring 2024-09-17T23:45:32.825 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2024-09-17T23:45:32.903 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-09-17T23:45:32.959 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '172.21.15.28')] 2024-09-17T23:45:32.960 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.28', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2024-09-17T23:45:32.961 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:32.961 DEBUG:teuthology.orchestra.run.smithi028:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-09-17T23:45:33.008 DEBUG:teuthology.orchestra.run.smithi028:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.28 --print /home/ubuntu/cephtest/cluster1.monmap 2024-09-17T23:45:33.089 INFO:teuthology.orchestra.run.smithi028.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2024-09-17T23:45:33.089 INFO:teuthology.orchestra.run.smithi028.stdout:monmaptool: generated fsid 57e7c284-39b0-4b91-954c-a87a8b8ca988 2024-09-17T23:45:33.089 INFO:teuthology.orchestra.run.smithi028.stdout:setting min_mon_release = quincy 2024-09-17T23:45:33.089 INFO:teuthology.orchestra.run.smithi028.stdout:epoch 0 2024-09-17T23:45:33.089 INFO:teuthology.orchestra.run.smithi028.stdout:fsid 57e7c284-39b0-4b91-954c-a87a8b8ca988 2024-09-17T23:45:33.089 INFO:teuthology.orchestra.run.smithi028.stdout:last_changed 2024-09-17T23:45:33.088826+0000 2024-09-17T23:45:33.089 INFO:teuthology.orchestra.run.smithi028.stdout:created 2024-09-17T23:45:33.088826+0000 2024-09-17T23:45:33.089 INFO:teuthology.orchestra.run.smithi028.stdout:min_mon_release 17 (quincy) 2024-09-17T23:45:33.089 INFO:teuthology.orchestra.run.smithi028.stdout:election_strategy: 1 2024-09-17T23:45:33.090 INFO:teuthology.orchestra.run.smithi028.stdout:0: [v2:172.21.15.28:3300/0,v1:172.21.15.28:6789/0] mon.a 2024-09-17T23:45:33.090 INFO:teuthology.orchestra.run.smithi028.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2024-09-17T23:45:33.091 DEBUG:teuthology.orchestra.run.smithi028:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-09-17T23:45:33.137 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 57e7c284-39b0-4b91-954c-a87a8b8ca988... 2024-09-17T23:45:33.137 DEBUG:teuthology.orchestra.run.smithi028:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2024-09-17T23:45:33.181 DEBUG:teuthology.orchestra.run.smithi049:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2024-09-17T23:45:33.209 INFO:teuthology.orchestra.run.smithi028.stdout:[global] 2024-09-17T23:45:33.210 INFO:teuthology.orchestra.run.smithi028.stdout: chdir = "" 2024-09-17T23:45:33.210 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-17T23:45:33.210 INFO:teuthology.orchestra.run.smithi028.stdout: auth supported = cephx 2024-09-17T23:45:33.210 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.210 INFO:teuthology.orchestra.run.smithi028.stdout: filestore xattr use omap = true 2024-09-17T23:45:33.210 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.210 INFO:teuthology.orchestra.run.smithi028.stdout: mon clock drift allowed = 1.000 2024-09-17T23:45:33.210 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.210 INFO:teuthology.orchestra.run.smithi028.stdout: osd crush chooseleaf type = 0 2024-09-17T23:45:33.210 INFO:teuthology.orchestra.run.smithi028.stdout: auth debug = true 2024-09-17T23:45:33.210 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: ms die on old message = true 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: ms die on bug = true 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: mon pg warn max object skew = 0 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: # disable pg_autoscaler by default for new pools 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: osd pool default size = 2 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd allow primary affinity = true 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd allow pg remap = true 2024-09-17T23:45:33.211 INFO:teuthology.orchestra.run.smithi028.stdout: mon warn on legacy crush tunables = false 2024-09-17T23:45:33.212 INFO:teuthology.orchestra.run.smithi028.stdout: mon warn on crush straw calc version zero = false 2024-09-17T23:45:33.212 INFO:teuthology.orchestra.run.smithi028.stdout: mon warn on no sortbitwise = false 2024-09-17T23:45:33.212 INFO:teuthology.orchestra.run.smithi028.stdout: mon warn on osd down out interval zero = false 2024-09-17T23:45:33.212 INFO:teuthology.orchestra.run.smithi028.stdout: mon warn on too few osds = false 2024-09-17T23:45:33.212 INFO:teuthology.orchestra.run.smithi028.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-17T23:45:33.212 INFO:teuthology.orchestra.run.smithi028.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-17T23:45:33.212 INFO:teuthology.orchestra.run.smithi028.stdout: mon_allow_pool_size_one = true 2024-09-17T23:45:33.212 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.212 INFO:teuthology.orchestra.run.smithi028.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-17T23:45:33.212 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.213 INFO:teuthology.orchestra.run.smithi028.stdout: osd default data pool replay window = 5 2024-09-17T23:45:33.213 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.213 INFO:teuthology.orchestra.run.smithi028.stdout: mon allow pool delete = true 2024-09-17T23:45:33.213 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.213 INFO:teuthology.orchestra.run.smithi028.stdout: mon cluster log file level = debug 2024-09-17T23:45:33.213 INFO:teuthology.orchestra.run.smithi028.stdout: debug asserts on shutdown = true 2024-09-17T23:45:33.213 INFO:teuthology.orchestra.run.smithi028.stdout: mon health detail to clog = false 2024-09-17T23:45:33.213 INFO:teuthology.orchestra.run.smithi028.stdout: mon host = 172.21.15.28 2024-09-17T23:45:33.213 INFO:teuthology.orchestra.run.smithi028.stdout: mon client directed command retry = 5 2024-09-17T23:45:33.213 INFO:teuthology.orchestra.run.smithi028.stdout: mon warn on pool no app = False 2024-09-17T23:45:33.213 INFO:teuthology.orchestra.run.smithi028.stdout: ms inject socket failures = 5000 2024-09-17T23:45:33.213 INFO:teuthology.orchestra.run.smithi028.stdout: fsid = 57e7c284-39b0-4b91-954c-a87a8b8ca988 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout:[osd] 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: osd journal size = 100 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: osd scrub load threshold = 5.0 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: osd scrub max interval = 600 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: osd mclock profile = high_recovery_ops 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: osd recover clone overlap = true 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: osd recovery max chunk = 1048576 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: osd debug shutdown = true 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: osd debug op order = true 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: osd debug verify stray on activate = true 2024-09-17T23:45:33.214 INFO:teuthology.orchestra.run.smithi028.stdout: osd debug trim objects = true 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: osd open classes on start = true 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: osd debug pg log writeout = true 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: osd deep scrub update digest min age = 30 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: osd map max advance = 10 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: journal zero on create = true 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: filestore ondisk finisher threads = 3 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: filestore apply finisher threads = 3 2024-09-17T23:45:33.215 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: bdev debug aio = true 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: osd debug misdirected ops = true 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: bluefs allocator = hybrid 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: bluestore allocator = hybrid 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: bluestore block size = 96636764160 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: bluestore fsck on mount = True 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: debug bluefs = 1/20 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: debug bluestore = 1/20 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: debug ms = 1 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: debug osd = 20 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: debug rocksdb = 4/10 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd backfillfull_ratio = 0.85 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd full ratio = 0.9 2024-09-17T23:45:33.216 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd nearfull ratio = 0.8 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout: osd failsafe full ratio = 0.95 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout: osd objectstore = bluestore 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout:[mgr] 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout: debug ms = 1 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout: debug mgr = 20 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout: debug mon = 20 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout: debug auth = 20 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout: mon reweight min pgs per osd = 4 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout: mon reweight min bytes per osd = 10 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout: mgr/telemetry/nag = false 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout:[mon] 2024-09-17T23:45:33.217 INFO:teuthology.orchestra.run.smithi028.stdout: debug ms = 1 2024-09-17T23:45:33.218 INFO:teuthology.orchestra.run.smithi028.stdout: debug mon = 20 2024-09-17T23:45:33.218 INFO:teuthology.orchestra.run.smithi028.stdout: debug paxos = 20 2024-09-17T23:45:33.218 INFO:teuthology.orchestra.run.smithi028.stdout: debug auth = 20 2024-09-17T23:45:33.218 INFO:teuthology.orchestra.run.smithi028.stdout: mon data avail warn = 5 2024-09-17T23:45:33.218 INFO:teuthology.orchestra.run.smithi028.stdout: mon mgr mkfs grace = 240 2024-09-17T23:45:33.218 INFO:teuthology.orchestra.run.smithi028.stdout: mon reweight min pgs per osd = 4 2024-09-17T23:45:33.218 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd reporter subtree level = osd 2024-09-17T23:45:33.218 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd prime pg temp = true 2024-09-17T23:45:33.218 INFO:teuthology.orchestra.run.smithi028.stdout: mon reweight min bytes per osd = 10 2024-09-17T23:45:33.218 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: auth mon ticket ttl = 660 # 11m 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: auth service ticket ttl = 240 # 4m 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: # don't complain about insecure global_id in the test suite 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: # 1m isn't quite enough 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: mon_down_mkfs_grace = 2m 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: mon_warn_on_filestore_osds = false 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:33.219 INFO:teuthology.orchestra.run.smithi028.stdout:[client] 2024-09-17T23:45:33.220 INFO:teuthology.orchestra.run.smithi028.stdout: rgw cache enabled = true 2024-09-17T23:45:33.220 INFO:teuthology.orchestra.run.smithi028.stdout: rgw enable ops log = true 2024-09-17T23:45:33.220 INFO:teuthology.orchestra.run.smithi028.stdout: rgw enable usage log = true 2024-09-17T23:45:33.220 INFO:teuthology.orchestra.run.smithi028.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-17T23:45:33.220 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-17T23:45:33.220 INFO:teuthology.orchestra.run.smithi028.stdout: debug rbd = 20 2024-09-17T23:45:33.220 INFO:teuthology.orchestra.run.smithi028.stdout: debug rbd_mirror = 15 2024-09-17T23:45:33.220 INFO:teuthology.orchestra.run.smithi028.stdout: log to stderr = False 2024-09-17T23:45:33.220 INFO:teuthology.orchestra.run.smithi028.stdout: rbd default features = 125 2024-09-17T23:45:33.220 INFO:teuthology.orchestra.run.smithi028.stdout:[mon.a] 2024-09-17T23:45:33.220 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.0] 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.1] 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.2] 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.3] 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.4] 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.221 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.222 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.5] 2024-09-17T23:45:33.222 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.222 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.222 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.6] 2024-09-17T23:45:33.222 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.222 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.222 INFO:teuthology.orchestra.run.smithi049.stdout:[global] 2024-09-17T23:45:33.223 INFO:teuthology.orchestra.run.smithi049.stdout: chdir = "" 2024-09-17T23:45:33.223 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-17T23:45:33.223 INFO:teuthology.orchestra.run.smithi049.stdout: auth supported = cephx 2024-09-17T23:45:33.223 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.223 INFO:teuthology.orchestra.run.smithi049.stdout: filestore xattr use omap = true 2024-09-17T23:45:33.223 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.223 INFO:teuthology.orchestra.run.smithi049.stdout: mon clock drift allowed = 1.000 2024-09-17T23:45:33.223 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.223 INFO:teuthology.orchestra.run.smithi049.stdout: osd crush chooseleaf type = 0 2024-09-17T23:45:33.223 INFO:teuthology.orchestra.run.smithi049.stdout: auth debug = true 2024-09-17T23:45:33.224 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.224 INFO:teuthology.orchestra.run.smithi049.stdout: ms die on old message = true 2024-09-17T23:45:33.224 INFO:teuthology.orchestra.run.smithi049.stdout: ms die on bug = true 2024-09-17T23:45:33.224 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.224 INFO:teuthology.orchestra.run.smithi049.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-17T23:45:33.224 INFO:teuthology.orchestra.run.smithi049.stdout: mon pg warn max object skew = 0 2024-09-17T23:45:33.224 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.224 INFO:teuthology.orchestra.run.smithi049.stdout: # disable pg_autoscaler by default for new pools 2024-09-17T23:45:33.224 INFO:teuthology.orchestra.run.smithi049.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-17T23:45:33.224 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.224 INFO:teuthology.orchestra.run.smithi049.stdout: osd pool default size = 2 2024-09-17T23:45:33.225 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.225 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd allow primary affinity = true 2024-09-17T23:45:33.225 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd allow pg remap = true 2024-09-17T23:45:33.225 INFO:teuthology.orchestra.run.smithi049.stdout: mon warn on legacy crush tunables = false 2024-09-17T23:45:33.225 INFO:teuthology.orchestra.run.smithi049.stdout: mon warn on crush straw calc version zero = false 2024-09-17T23:45:33.226 INFO:teuthology.orchestra.run.smithi049.stdout: mon warn on no sortbitwise = false 2024-09-17T23:45:33.226 INFO:teuthology.orchestra.run.smithi049.stdout: mon warn on osd down out interval zero = false 2024-09-17T23:45:33.226 INFO:teuthology.orchestra.run.smithi049.stdout: mon warn on too few osds = false 2024-09-17T23:45:33.226 INFO:teuthology.orchestra.run.smithi049.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-17T23:45:33.226 INFO:teuthology.orchestra.run.smithi049.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-17T23:45:33.226 INFO:teuthology.orchestra.run.smithi049.stdout: mon_allow_pool_size_one = true 2024-09-17T23:45:33.226 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.226 INFO:teuthology.orchestra.run.smithi049.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-17T23:45:33.226 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.226 INFO:teuthology.orchestra.run.smithi049.stdout: osd default data pool replay window = 5 2024-09-17T23:45:33.226 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.226 INFO:teuthology.orchestra.run.smithi049.stdout: mon allow pool delete = true 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: mon cluster log file level = debug 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: debug asserts on shutdown = true 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: mon health detail to clog = false 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: mon host = 172.21.15.28 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: mon client directed command retry = 5 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: mon warn on pool no app = False 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: ms inject socket failures = 5000 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: fsid = 57e7c284-39b0-4b91-954c-a87a8b8ca988 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout:[osd] 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: osd journal size = 100 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: osd scrub load threshold = 5.0 2024-09-17T23:45:33.227 INFO:teuthology.orchestra.run.smithi049.stdout: osd scrub max interval = 600 2024-09-17T23:45:33.228 INFO:teuthology.orchestra.run.smithi049.stdout: osd mclock profile = high_recovery_ops 2024-09-17T23:45:33.228 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.228 INFO:teuthology.orchestra.run.smithi049.stdout: osd recover clone overlap = true 2024-09-17T23:45:33.228 INFO:teuthology.orchestra.run.smithi049.stdout: osd recovery max chunk = 1048576 2024-09-17T23:45:33.228 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.228 INFO:teuthology.orchestra.run.smithi049.stdout: osd debug shutdown = true 2024-09-17T23:45:33.228 INFO:teuthology.orchestra.run.smithi049.stdout: osd debug op order = true 2024-09-17T23:45:33.228 INFO:teuthology.orchestra.run.smithi049.stdout: osd debug verify stray on activate = true 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: osd debug trim objects = true 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: osd open classes on start = true 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: osd debug pg log writeout = true 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: osd deep scrub update digest min age = 30 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: osd map max advance = 10 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: journal zero on create = true 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: filestore ondisk finisher threads = 3 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: filestore apply finisher threads = 3 2024-09-17T23:45:33.229 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: bdev debug aio = true 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: osd debug misdirected ops = true 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: bluefs allocator = hybrid 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: bluestore allocator = hybrid 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: bluestore block size = 96636764160 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: bluestore fsck on mount = True 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: debug bluefs = 1/20 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: debug bluestore = 1/20 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: debug ms = 1 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: debug osd = 20 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: debug rocksdb = 4/10 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd backfillfull_ratio = 0.85 2024-09-17T23:45:33.230 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd full ratio = 0.9 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd nearfull ratio = 0.8 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout: osd failsafe full ratio = 0.95 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout: osd objectstore = bluestore 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout:[mgr] 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout: debug ms = 1 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout: debug mgr = 20 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout: debug mon = 20 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout: debug auth = 20 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout: mon reweight min pgs per osd = 4 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout: mon reweight min bytes per osd = 10 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout: mgr/telemetry/nag = false 2024-09-17T23:45:33.231 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout:[mon] 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout: debug ms = 1 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout: debug mon = 20 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout: debug paxos = 20 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout: debug auth = 20 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout: mon data avail warn = 5 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout: mon mgr mkfs grace = 240 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout: mon reweight min pgs per osd = 4 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd reporter subtree level = osd 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd prime pg temp = true 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout: mon reweight min bytes per osd = 10 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.232 INFO:teuthology.orchestra.run.smithi049.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-17T23:45:33.233 INFO:teuthology.orchestra.run.smithi049.stdout: auth mon ticket ttl = 660 # 11m 2024-09-17T23:45:33.233 INFO:teuthology.orchestra.run.smithi049.stdout: auth service ticket ttl = 240 # 4m 2024-09-17T23:45:33.233 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.233 INFO:teuthology.orchestra.run.smithi049.stdout: # don't complain about insecure global_id in the test suite 2024-09-17T23:45:33.233 INFO:teuthology.orchestra.run.smithi049.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-17T23:45:33.233 INFO:teuthology.orchestra.run.smithi049.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-17T23:45:33.233 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.233 INFO:teuthology.orchestra.run.smithi049.stdout: # 1m isn't quite enough 2024-09-17T23:45:33.233 INFO:teuthology.orchestra.run.smithi049.stdout: mon_down_mkfs_grace = 2m 2024-09-17T23:45:33.233 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.233 INFO:teuthology.orchestra.run.smithi049.stdout: mon_warn_on_filestore_osds = false 2024-09-17T23:45:33.233 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout:[client] 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout: rgw cache enabled = true 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout: rgw enable ops log = true 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout: rgw enable usage log = true 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout: debug rbd = 20 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout: debug rbd_mirror = 15 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout: log to stderr = False 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout: rbd default features = 125 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout:[mon.a] 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.0] 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.234 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.235 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.1] 2024-09-17T23:45:33.235 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.235 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.235 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.2] 2024-09-17T23:45:33.235 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.235 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.235 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.3] 2024-09-17T23:45:33.235 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.235 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.235 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.4] 2024-09-17T23:45:33.235 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.235 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.236 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.5] 2024-09-17T23:45:33.236 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.236 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.236 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.6] 2024-09-17T23:45:33.236 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:45:33.236 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:45:33.238 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2024-09-17T23:45:33.238 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2024-09-17T23:45:33.331 INFO:tasks.ceph:Copying monmap to all nodes... 2024-09-17T23:45:33.331 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:33.332 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2024-09-17T23:45:33.338 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:33.339 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2024-09-17T23:45:33.385 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi028.front.sepia.ceph.com 2024-09-17T23:45:33.385 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:33.385 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/ceph/cluster1.keyring 2024-09-17T23:45:33.386 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-09-17T23:45:33.449 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:33.449 DEBUG:teuthology.orchestra.run.smithi028:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2024-09-17T23:45:33.497 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi049.front.sepia.ceph.com 2024-09-17T23:45:33.498 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:33.498 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd of=/etc/ceph/cluster1.keyring 2024-09-17T23:45:33.498 DEBUG:teuthology.orchestra.run.smithi049:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-09-17T23:45:33.522 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:33.523 DEBUG:teuthology.orchestra.run.smithi049:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2024-09-17T23:45:33.574 INFO:tasks.ceph:Setting up mon nodes... 2024-09-17T23:45:33.574 INFO:tasks.ceph:Setting up mgr nodes... 2024-09-17T23:45:33.574 DEBUG:teuthology.orchestra.run.smithi028:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2024-09-17T23:45:33.620 INFO:teuthology.orchestra.run.smithi028.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2024-09-17T23:45:33.623 INFO:tasks.ceph:Setting up mds nodes... 2024-09-17T23:45:33.623 INFO:tasks.ceph_client:Setting up client nodes... 2024-09-17T23:45:33.624 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2024-09-17T23:45:33.658 INFO:teuthology.orchestra.run.smithi028.stdout:creating /etc/ceph/cluster1.client.0.keyring 2024-09-17T23:45:33.669 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2024-09-17T23:45:33.706 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2024-09-17T23:45:33.738 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2024-09-17T23:45:33.819 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2024-09-17T23:45:33.830 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2024-09-17T23:45:33.909 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2024-09-17T23:45:33.921 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2024-09-17T23:45:34.002 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2024-09-17T23:45:34.012 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2024-09-17T23:45:34.084 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2024-09-17T23:45:34.094 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2024-09-17T23:45:34.167 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2024-09-17T23:45:34.177 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2024-09-17T23:45:34.253 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2024-09-17T23:45:34.262 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2024-09-17T23:45:34.337 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2024-09-17T23:45:34.348 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-09-17T23:45:34.348 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi028.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-17T23:45:34.348 DEBUG:teuthology.orchestra.run.smithi028:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2024-09-17T23:45:34.362 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-17T23:45:34.362 INFO:tasks.ceph:role: cluster1.osd.0 2024-09-17T23:45:34.362 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi028.front.sepia.ceph.com 2024-09-17T23:45:34.362 DEBUG:teuthology.orchestra.run.smithi028:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-09-17T23:45:34.416 INFO:teuthology.orchestra.run.smithi028.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-09-17T23:45:34.417 INFO:teuthology.orchestra.run.smithi028.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-17T23:45:34.417 INFO:teuthology.orchestra.run.smithi028.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-17T23:45:34.417 INFO:teuthology.orchestra.run.smithi028.stdout: = reflink=1 bigtime=0 inobtcount=0 2024-09-17T23:45:34.417 INFO:teuthology.orchestra.run.smithi028.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-17T23:45:34.417 INFO:teuthology.orchestra.run.smithi028.stdout: = sunit=0 swidth=0 blks 2024-09-17T23:45:34.417 INFO:teuthology.orchestra.run.smithi028.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-17T23:45:34.417 INFO:teuthology.orchestra.run.smithi028.stdout:log =internal log bsize=4096 blocks=11443, version=2 2024-09-17T23:45:34.417 INFO:teuthology.orchestra.run.smithi028.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-17T23:45:34.417 INFO:teuthology.orchestra.run.smithi028.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-17T23:45:34.440 INFO:teuthology.orchestra.run.smithi028.stdout:Discarding blocks...Done. 2024-09-17T23:45:34.442 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi028.front.sepia.ceph.com -o noatime 2024-09-17T23:45:34.442 DEBUG:teuthology.orchestra.run.smithi028:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2024-09-17T23:45:34.500 DEBUG:teuthology.orchestra.run.smithi028:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2024-09-17T23:45:34.552 INFO:teuthology.orchestra.run.smithi028.stderr:sudo: /sbin/restorecon: command not found 2024-09-17T23:45:34.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T23:45:34.553 DEBUG:teuthology.orchestra.run.smithi028:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2024-09-17T23:45:34.608 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-17T23:45:34.608 INFO:tasks.ceph:role: cluster1.osd.1 2024-09-17T23:45:34.608 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi028.front.sepia.ceph.com 2024-09-17T23:45:34.608 DEBUG:teuthology.orchestra.run.smithi028:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-09-17T23:45:34.665 INFO:teuthology.orchestra.run.smithi028.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-09-17T23:45:34.665 INFO:teuthology.orchestra.run.smithi028.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-17T23:45:34.665 INFO:teuthology.orchestra.run.smithi028.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-17T23:45:34.665 INFO:teuthology.orchestra.run.smithi028.stdout: = reflink=1 bigtime=0 inobtcount=0 2024-09-17T23:45:34.665 INFO:teuthology.orchestra.run.smithi028.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-17T23:45:34.666 INFO:teuthology.orchestra.run.smithi028.stdout: = sunit=0 swidth=0 blks 2024-09-17T23:45:34.666 INFO:teuthology.orchestra.run.smithi028.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-17T23:45:34.666 INFO:teuthology.orchestra.run.smithi028.stdout:log =internal log bsize=4096 blocks=11443, version=2 2024-09-17T23:45:34.666 INFO:teuthology.orchestra.run.smithi028.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-17T23:45:34.666 INFO:teuthology.orchestra.run.smithi028.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-17T23:45:34.689 INFO:teuthology.orchestra.run.smithi028.stdout:Discarding blocks...Done. 2024-09-17T23:45:34.690 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi028.front.sepia.ceph.com -o noatime 2024-09-17T23:45:34.690 DEBUG:teuthology.orchestra.run.smithi028:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2024-09-17T23:45:34.748 DEBUG:teuthology.orchestra.run.smithi028:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2024-09-17T23:45:34.800 INFO:teuthology.orchestra.run.smithi028.stderr:sudo: /sbin/restorecon: command not found 2024-09-17T23:45:34.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T23:45:34.801 DEBUG:teuthology.orchestra.run.smithi028:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2024-09-17T23:45:34.853 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-17T23:45:34.853 INFO:tasks.ceph:role: cluster1.osd.2 2024-09-17T23:45:34.854 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi028.front.sepia.ceph.com 2024-09-17T23:45:34.854 DEBUG:teuthology.orchestra.run.smithi028:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-09-17T23:45:34.908 INFO:teuthology.orchestra.run.smithi028.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-09-17T23:45:34.909 INFO:teuthology.orchestra.run.smithi028.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-17T23:45:34.909 INFO:teuthology.orchestra.run.smithi028.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-17T23:45:34.909 INFO:teuthology.orchestra.run.smithi028.stdout: = reflink=1 bigtime=0 inobtcount=0 2024-09-17T23:45:34.909 INFO:teuthology.orchestra.run.smithi028.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-17T23:45:34.909 INFO:teuthology.orchestra.run.smithi028.stdout: = sunit=0 swidth=0 blks 2024-09-17T23:45:34.909 INFO:teuthology.orchestra.run.smithi028.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-17T23:45:34.910 INFO:teuthology.orchestra.run.smithi028.stdout:log =internal log bsize=4096 blocks=11443, version=2 2024-09-17T23:45:34.910 INFO:teuthology.orchestra.run.smithi028.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-17T23:45:34.910 INFO:teuthology.orchestra.run.smithi028.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-17T23:45:34.933 INFO:teuthology.orchestra.run.smithi028.stdout:Discarding blocks...Done. 2024-09-17T23:45:34.935 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi028.front.sepia.ceph.com -o noatime 2024-09-17T23:45:34.935 DEBUG:teuthology.orchestra.run.smithi028:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2024-09-17T23:45:34.993 DEBUG:teuthology.orchestra.run.smithi028:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2024-09-17T23:45:35.049 INFO:teuthology.orchestra.run.smithi028.stderr:sudo: /sbin/restorecon: command not found 2024-09-17T23:45:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T23:45:35.050 DEBUG:teuthology.orchestra.run.smithi028:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-09-17T23:45:35.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-09-17T23:45:35.167+0000 7f96aa8b17c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2024-09-17T23:45:35.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-09-17T23:45:35.171+0000 7f96aa8b17c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2024-09-17T23:45:35.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-09-17T23:45:35.171+0000 7f96aa8b17c0 -1 bdev(0x557eaaabd800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2024-09-17T23:45:35.172 INFO:teuthology.orchestra.run.smithi028.stderr:2024-09-17T23:45:35.171+0000 7f96aa8b17c0 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2024-09-17T23:45:38.322 DEBUG:teuthology.orchestra.run.smithi028:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2024-09-17T23:45:38.337 DEBUG:teuthology.orchestra.run.smithi028:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-09-17T23:45:38.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-09-17T23:45:38.415+0000 7f0878bbf7c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2024-09-17T23:45:38.419 INFO:teuthology.orchestra.run.smithi028.stderr:2024-09-17T23:45:38.419+0000 7f0878bbf7c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2024-09-17T23:45:38.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-09-17T23:45:38.419+0000 7f0878bbf7c0 -1 bdev(0x5575430a5800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2024-09-17T23:45:38.420 INFO:teuthology.orchestra.run.smithi028.stderr:2024-09-17T23:45:38.419+0000 7f0878bbf7c0 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2024-09-17T23:45:41.551 DEBUG:teuthology.orchestra.run.smithi028:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2024-09-17T23:45:41.609 DEBUG:teuthology.orchestra.run.smithi028:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-09-17T23:45:41.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-09-17T23:45:41.687+0000 7f147c4d97c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2024-09-17T23:45:41.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-09-17T23:45:41.691+0000 7f147c4d97c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2024-09-17T23:45:41.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-09-17T23:45:41.691+0000 7f147c4d97c0 -1 bdev(0x562ea4d95800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2024-09-17T23:45:41.692 INFO:teuthology.orchestra.run.smithi028.stderr:2024-09-17T23:45:41.691+0000 7f147c4d97c0 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2024-09-17T23:45:44.847 DEBUG:teuthology.orchestra.run.smithi028:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2024-09-17T23:45:44.905 INFO:tasks.ceph:Reading keys from all nodes... 2024-09-17T23:45:44.905 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:44.905 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2024-09-17T23:45:44.959 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:44.959 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2024-09-17T23:45:45.015 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:45.015 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2024-09-17T23:45:45.071 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:45.071 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2024-09-17T23:45:45.132 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:45.132 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2024-09-17T23:45:45.180 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:45.180 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2024-09-17T23:45:45.186 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:45.187 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2024-09-17T23:45:45.233 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:45.234 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2024-09-17T23:45:45.281 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:45.281 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2024-09-17T23:45:45.329 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:45.329 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2024-09-17T23:45:45.377 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:45.377 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2024-09-17T23:45:45.425 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:45.426 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2024-09-17T23:45:45.473 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:45:45.473 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2024-09-17T23:45:45.521 INFO:tasks.ceph:Adding keys to all mons... 2024-09-17T23:45:45.521 DEBUG:teuthology.orchestra.run.smithi028:> sudo tee -a /etc/ceph/cluster1.keyring 2024-09-17T23:45:45.534 INFO:teuthology.orchestra.run.smithi028.stdout:[mgr.x] 2024-09-17T23:45:45.534 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQCdFOpm+3f7JBAAYuW+HOMS8f8Fp33PkBm/tw== 2024-09-17T23:45:45.534 INFO:teuthology.orchestra.run.smithi028.stdout:[osd.0] 2024-09-17T23:45:45.534 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQCfFOpmWH85ChAAjEBeIb6pqwT3Wld7mRT1/g== 2024-09-17T23:45:45.534 INFO:teuthology.orchestra.run.smithi028.stdout:[osd.1] 2024-09-17T23:45:45.534 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQCiFOpmS7T+GBAARQVYGuuC+QUNDOotIGe4Dw== 2024-09-17T23:45:45.534 INFO:teuthology.orchestra.run.smithi028.stdout:[osd.2] 2024-09-17T23:45:45.534 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQClFOpm0t85KRAAzJJ3u+QYU19U1YgEYBBh4Q== 2024-09-17T23:45:45.534 INFO:teuthology.orchestra.run.smithi028.stdout:[client.0] 2024-09-17T23:45:45.535 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQCdFOpmvxxEJxAAVLdpUXyB+ZKfLBoaD0BdUw== 2024-09-17T23:45:45.535 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror] 2024-09-17T23:45:45.535 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQCdFOpmfZUbKhAABdKNXVO69OAlJyNDh+gWUQ== 2024-09-17T23:45:45.535 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.0] 2024-09-17T23:45:45.535 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQCdFOpmRvXQMBAAA4FZo1xEvCA4a7SPOTvrWg== 2024-09-17T23:45:45.535 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.1] 2024-09-17T23:45:45.535 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQCdFOpmT0oyNhAAu934BeTq3SVoVsmdP7g4NQ== 2024-09-17T23:45:45.535 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.2] 2024-09-17T23:45:45.535 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQCeFOpm788eABAAI9X0nomBDNFK8kbVqfVR0Q== 2024-09-17T23:45:45.535 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.3] 2024-09-17T23:45:45.535 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQCeFOpmGNYHBRAAf0ZyH89sy5DaccfqeUcSaA== 2024-09-17T23:45:45.535 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.4] 2024-09-17T23:45:45.536 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQCeFOpm51rzCRAAnK+BWXuAqGmv88f0inSWVQ== 2024-09-17T23:45:45.536 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.5] 2024-09-17T23:45:45.536 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQCeFOpmrp0RDxAA1oLKpYO+MzrMg6eFwdxmRg== 2024-09-17T23:45:45.536 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.6] 2024-09-17T23:45:45.536 INFO:teuthology.orchestra.run.smithi028.stdout: key = AQCeFOpmYosVFBAADkGFhBcbaGjVa7XeKm2+hw== 2024-09-17T23:45:45.537 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-17T23:45:45.627 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-17T23:45:45.670 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-17T23:45:45.755 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-17T23:45:45.844 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:45:45.892 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:45:45.982 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:45:46.030 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:45:46.074 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:45:46.157 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:45:46.249 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:45:46.294 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:45:46.385 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:45:46.430 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-09-17T23:45:46.430 DEBUG:teuthology.orchestra.run.smithi028:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2024-09-17T23:45:46.483 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2024-09-17T23:45:46.580 DEBUG:teuthology.orchestra.run.smithi028:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2024-09-17T23:45:46.590 DEBUG:teuthology.orchestra.run.smithi028:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2024-09-17T23:45:46.635 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2024-09-17T23:45:46.635 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2024-09-17T23:45:46.635 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2024-09-17T23:45:46.681 INFO:tasks.ceph.cluster1.mon.a:Started 2024-09-17T23:45:46.681 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2024-09-17T23:45:46.681 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2024-09-17T23:45:46.681 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2024-09-17T23:45:46.683 INFO:tasks.ceph.cluster1.mgr.x:Started 2024-09-17T23:45:46.683 INFO:tasks.ceph:Setting crush tunables to default 2024-09-17T23:45:46.683 DEBUG:teuthology.orchestra.run.smithi028:> sudo ceph --cluster cluster1 osd crush tunables default 2024-09-17T23:45:46.899 INFO:teuthology.orchestra.run.smithi028.stderr:adjusted tunables profile to default 2024-09-17T23:45:46.917 INFO:tasks.ceph:check_enable_crimson: False 2024-09-17T23:45:46.917 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2024-09-17T23:45:46.918 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:46.918 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2024-09-17T23:45:46.930 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:46.930 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2024-09-17T23:45:46.985 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:45:46.986 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2024-09-17T23:45:47.036 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:47.035+0000 7fb5c8cc2180 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T23:45:47.042 DEBUG:teuthology.orchestra.run.smithi028:> sudo ceph --cluster cluster1 osd new 8e6f69c1-27ba-4644-9817-26dce7dd505c 0 2024-09-17T23:45:47.287 INFO:teuthology.orchestra.run.smithi028.stdout:0 2024-09-17T23:45:47.306 DEBUG:teuthology.orchestra.run.smithi028:> sudo ceph --cluster cluster1 osd new 9bff203c-5eea-4198-b867-3161de2c53d7 1 2024-09-17T23:45:47.508 INFO:teuthology.orchestra.run.smithi028.stdout:1 2024-09-17T23:45:47.525 DEBUG:teuthology.orchestra.run.smithi028:> sudo ceph --cluster cluster1 osd new 6fdba670-6e7a-4175-82f1-097f630f6148 2 2024-09-17T23:45:47.545 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:47.543+0000 7fb5c8cc2180 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T23:45:47.716 INFO:teuthology.orchestra.run.smithi028.stdout:2 2024-09-17T23:45:47.729 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:47.727+0000 7fb5c8cc2180 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T23:45:47.732 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2024-09-17T23:45:47.733 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2024-09-17T23:45:47.777 INFO:tasks.ceph.cluster1.osd.0:Started 2024-09-17T23:45:47.778 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2024-09-17T23:45:47.778 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2024-09-17T23:45:47.781 INFO:tasks.ceph.cluster1.osd.1:Started 2024-09-17T23:45:47.782 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2024-09-17T23:45:47.782 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2024-09-17T23:45:47.786 INFO:tasks.ceph.cluster1.osd.2:Started 2024-09-17T23:45:47.786 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-17T23:45:47.967 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:45:47.967 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":5,"fsid":"57e7c284-39b0-4b91-954c-a87a8b8ca988","created":"2024-09-17T23:45:46.808870+0000","modified":"2024-09-17T23:45:47.715296+0000","last_up_change":"0.000000","last_in_change":"2024-09-17T23:45:47.715296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"8e6f69c1-27ba-4644-9817-26dce7dd505c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"9bff203c-5eea-4198-b867-3161de2c53d7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"6fdba670-6e7a-4175-82f1-097f630f6148","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T23:45:47.982 INFO:tasks.ceph.ceph_manager.cluster1:[] 2024-09-17T23:45:47.982 INFO:tasks.ceph:Waiting for OSDs to come up 2024-09-17T23:45:48.285 DEBUG:teuthology.orchestra.run.smithi028:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-09-17T23:45:48.345 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:48.343+0000 7fb5c8cc2180 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T23:45:48.402 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:48.399+0000 7fb5c8cc2180 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T23:45:48.459 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:48.455+0000 7fb5c8cc2180 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T23:45:48.469 INFO:teuthology.misc.health.smithi028.stdout: 2024-09-17T23:45:48.469 INFO:teuthology.misc.health.smithi028.stdout:{"epoch":5,"fsid":"57e7c284-39b0-4b91-954c-a87a8b8ca988","created":"2024-09-17T23:45:46.808870+0000","modified":"2024-09-17T23:45:47.715296+0000","last_up_change":"0.000000","last_in_change":"2024-09-17T23:45:47.715296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"8e6f69c1-27ba-4644-9817-26dce7dd505c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"9bff203c-5eea-4198-b867-3161de2c53d7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"6fdba670-6e7a-4175-82f1-097f630f6148","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T23:45:48.485 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-17T23:45:48.542 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:48.539+0000 7fb5c8cc2180 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T23:45:48.596 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:48.595+0000 7fb5c8cc2180 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T23:45:48.677 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:48.675+0000 7fb5c8cc2180 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T23:45:48.733 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:48.731+0000 7fb5c8cc2180 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T23:45:48.892 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:48.891+0000 7fb5c8cc2180 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T23:45:48.956 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:48.955+0000 7fb5c8cc2180 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T23:45:49.101 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-17T23:45:49.099+0000 7fb0dfce77c0 -1 Falling back to public interface 2024-09-17T23:45:49.337 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-17T23:45:49.335+0000 7fac435bd7c0 -1 Falling back to public interface 2024-09-17T23:45:49.352 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-17T23:45:49.351+0000 7f84ac61e7c0 -1 Falling back to public interface 2024-09-17T23:45:49.604 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2024-09-17T23:45:49.604 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2024-09-17T23:45:49.604 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr: from numpy import show_config as show_numpy_config 2024-09-17T23:45:49.654 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:49.651+0000 7fb5c8cc2180 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T23:45:49.835 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:49.831+0000 7fb5c8cc2180 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T23:45:50.070 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:50.067+0000 7fb5c8cc2180 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T23:45:50.154 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:50.151+0000 7fb5c8cc2180 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T23:45:50.298 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:50.295+0000 7fb5c8cc2180 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T23:45:50.933 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:50.931+0000 7fb5c8cc2180 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T23:45:51.017 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:51.015+0000 7fb5c8cc2180 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T23:45:51.209 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:51.207+0000 7fb5c8cc2180 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T23:45:51.286 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:51.283+0000 7fb5c8cc2180 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T23:45:51.592 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:51.591+0000 7fb5c8cc2180 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T23:45:51.657 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:51.655+0000 7fb5c8cc2180 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T23:45:52.894 INFO:tasks.ceph.cluster1.mgr.x.smithi028.stderr:2024-09-17T23:45:52.891+0000 7fb56f7f8640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-17T23:45:53.116 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-17T23:45:53.115+0000 7fb0dfce77c0 -1 osd.2 0 log_to_monitors true 2024-09-17T23:45:53.309 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-17T23:45:53.307+0000 7f84ac61e7c0 -1 osd.0 0 log_to_monitors true 2024-09-17T23:45:53.317 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-17T23:45:53.315+0000 7fac435bd7c0 -1 osd.1 0 log_to_monitors true 2024-09-17T23:45:54.788 DEBUG:teuthology.orchestra.run.smithi028:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-09-17T23:45:55.183 INFO:teuthology.misc.health.smithi028.stdout: 2024-09-17T23:45:55.183 INFO:teuthology.misc.health.smithi028.stdout:{"epoch":7,"fsid":"57e7c284-39b0-4b91-954c-a87a8b8ca988","created":"2024-09-17T23:45:46.808870+0000","modified":"2024-09-17T23:45:54.688119+0000","last_up_change":"0.000000","last_in_change":"2024-09-17T23:45:47.715296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"8e6f69c1-27ba-4644-9817-26dce7dd505c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"9bff203c-5eea-4198-b867-3161de2c53d7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"6fdba670-6e7a-4175-82f1-097f630f6148","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T23:45:55.206 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-17T23:45:56.028 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-17T23:45:56.027+0000 7f84a8dd9640 -1 osd.0 0 waiting for initial osdmap 2024-09-17T23:45:56.056 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-17T23:45:56.055+0000 7f84a03fc640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T23:45:56.059 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-17T23:45:56.059+0000 7fb0dc4a2640 -1 osd.2 0 waiting for initial osdmap 2024-09-17T23:45:56.076 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-17T23:45:56.075+0000 7fac3f566640 -1 osd.1 0 waiting for initial osdmap 2024-09-17T23:45:56.079 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-17T23:45:56.079+0000 7fb0d3ac5640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T23:45:56.103 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-17T23:45:56.103+0000 7fac3739b640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T23:46:01.509 DEBUG:teuthology.orchestra.run.smithi028:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-09-17T23:46:01.819 INFO:teuthology.misc.health.smithi028.stdout: 2024-09-17T23:46:01.819 INFO:teuthology.misc.health.smithi028.stdout:{"epoch":11,"fsid":"57e7c284-39b0-4b91-954c-a87a8b8ca988","created":"2024-09-17T23:45:46.808870+0000","modified":"2024-09-17T23:46:00.032590+0000","last_up_change":"2024-09-17T23:45:57.028864+0000","last_in_change":"2024-09-17T23:45:47.715296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T23:45:57.757216+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"8e6f69c1-27ba-4644-9817-26dce7dd505c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6817","nonce":2136955652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6819","nonce":2136955652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6823","nonce":2136955652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6821","nonce":2136955652}]},"public_addr":"172.21.15.28:6817/2136955652","cluster_addr":"172.21.15.28:6819/2136955652","heartbeat_back_addr":"172.21.15.28:6823/2136955652","heartbeat_front_addr":"172.21.15.28:6821/2136955652","state":["exists","up"]},{"osd":1,"uuid":"9bff203c-5eea-4198-b867-3161de2c53d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6809","nonce":3505522461}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6811","nonce":3505522461}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6815","nonce":3505522461}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6813","nonce":3505522461}]},"public_addr":"172.21.15.28:6809/3505522461","cluster_addr":"172.21.15.28:6811/3505522461","heartbeat_back_addr":"172.21.15.28:6815/3505522461","heartbeat_front_addr":"172.21.15.28:6813/3505522461","state":["exists","up"]},{"osd":2,"uuid":"6fdba670-6e7a-4175-82f1-097f630f6148","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6800","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6801","nonce":182730647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6803","nonce":182730647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6807","nonce":182730647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6805","nonce":182730647}]},"public_addr":"172.21.15.28:6801/182730647","cluster_addr":"172.21.15.28:6803/182730647","heartbeat_back_addr":"172.21.15.28:6807/182730647","heartbeat_front_addr":"172.21.15.28:6805/182730647","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:45:54.289133+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:45:54.275284+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:45:54.091032+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T23:46:01.838 DEBUG:teuthology.misc:3 of 3 OSDs are up 2024-09-17T23:46:01.839 INFO:tasks.ceph:Creating RBD pool 2024-09-17T23:46:01.839 DEBUG:teuthology.orchestra.run.smithi028:> sudo ceph --cluster cluster1 osd pool create rbd 8 2024-09-17T23:46:02.714 INFO:teuthology.orchestra.run.smithi028.stderr:pool 'rbd' created 2024-09-17T23:46:02.744 DEBUG:teuthology.orchestra.run.smithi028:> rbd --cluster cluster1 pool init rbd 2024-09-17T23:46:05.758 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2024-09-17T23:46:05.758 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2024-09-17T23:46:05.758 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2024-09-17T23:46:05.759 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2024-09-17T23:46:05.759 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-09-17T23:46:06.113 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:06.131 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":5,"flags":0,"active_gid":4099,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6824","nonce":2486455888},{"type":"v1","addr":"172.21.15.28:6825","nonce":2486455888}]},"active_addr":"172.21.15.28:6825/2486455888","active_change":"2024-09-17T23:45:51.677894+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":2840096636}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":1435846870}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":4162648526}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":2892548590}]}]} 2024-09-17T23:46:06.133 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2024-09-17T23:46:06.133 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2024-09-17T23:46:06.133 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-17T23:46:06.415 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:06.415 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":15,"fsid":"57e7c284-39b0-4b91-954c-a87a8b8ca988","created":"2024-09-17T23:45:46.808870+0000","modified":"2024-09-17T23:46:05.724963+0000","last_up_change":"2024-09-17T23:45:57.028864+0000","last_in_change":"2024-09-17T23:45:47.715296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T23:45:57.757216+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-17T23:46:02.187572+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"8e6f69c1-27ba-4644-9817-26dce7dd505c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6817","nonce":2136955652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6819","nonce":2136955652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6823","nonce":2136955652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6821","nonce":2136955652}]},"public_addr":"172.21.15.28:6817/2136955652","cluster_addr":"172.21.15.28:6819/2136955652","heartbeat_back_addr":"172.21.15.28:6823/2136955652","heartbeat_front_addr":"172.21.15.28:6821/2136955652","state":["exists","up"]},{"osd":1,"uuid":"9bff203c-5eea-4198-b867-3161de2c53d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6809","nonce":3505522461}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6811","nonce":3505522461}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6815","nonce":3505522461}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6813","nonce":3505522461}]},"public_addr":"172.21.15.28:6809/3505522461","cluster_addr":"172.21.15.28:6811/3505522461","heartbeat_back_addr":"172.21.15.28:6815/3505522461","heartbeat_front_addr":"172.21.15.28:6813/3505522461","state":["exists","up"]},{"osd":2,"uuid":"6fdba670-6e7a-4175-82f1-097f630f6148","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6800","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6801","nonce":182730647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6803","nonce":182730647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6807","nonce":182730647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6805","nonce":182730647}]},"public_addr":"172.21.15.28:6801/182730647","cluster_addr":"172.21.15.28:6803/182730647","heartbeat_back_addr":"172.21.15.28:6807/182730647","heartbeat_front_addr":"172.21.15.28:6805/182730647","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:45:54.289133+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:45:54.275284+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:45:54.091032+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T23:46:06.433 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2024-09-17T23:46:06.433 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-17T23:46:06.717 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:06.717 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":15,"fsid":"57e7c284-39b0-4b91-954c-a87a8b8ca988","created":"2024-09-17T23:45:46.808870+0000","modified":"2024-09-17T23:46:05.724963+0000","last_up_change":"2024-09-17T23:45:57.028864+0000","last_in_change":"2024-09-17T23:45:47.715296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T23:45:57.757216+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-17T23:46:02.187572+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"8e6f69c1-27ba-4644-9817-26dce7dd505c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6817","nonce":2136955652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6819","nonce":2136955652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6823","nonce":2136955652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6821","nonce":2136955652}]},"public_addr":"172.21.15.28:6817/2136955652","cluster_addr":"172.21.15.28:6819/2136955652","heartbeat_back_addr":"172.21.15.28:6823/2136955652","heartbeat_front_addr":"172.21.15.28:6821/2136955652","state":["exists","up"]},{"osd":1,"uuid":"9bff203c-5eea-4198-b867-3161de2c53d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6809","nonce":3505522461}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6811","nonce":3505522461}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6815","nonce":3505522461}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6813","nonce":3505522461}]},"public_addr":"172.21.15.28:6809/3505522461","cluster_addr":"172.21.15.28:6811/3505522461","heartbeat_back_addr":"172.21.15.28:6815/3505522461","heartbeat_front_addr":"172.21.15.28:6813/3505522461","state":["exists","up"]},{"osd":2,"uuid":"6fdba670-6e7a-4175-82f1-097f630f6148","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6800","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6801","nonce":182730647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6803","nonce":182730647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6807","nonce":182730647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6805","nonce":182730647}]},"public_addr":"172.21.15.28:6801/182730647","cluster_addr":"172.21.15.28:6803/182730647","heartbeat_back_addr":"172.21.15.28:6807/182730647","heartbeat_front_addr":"172.21.15.28:6805/182730647","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:45:54.289133+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:45:54.275284+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:45:54.091032+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T23:46:06.735 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2024-09-17T23:46:06.736 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2024-09-17T23:46:06.737 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2024-09-17T23:46:06.890 INFO:teuthology.orchestra.run.smithi028.stdout:34359738372 2024-09-17T23:46:06.891 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2024-09-17T23:46:06.896 INFO:teuthology.orchestra.run.smithi028.stdout:34359738372 2024-09-17T23:46:06.896 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2024-09-17T23:46:06.903 INFO:teuthology.orchestra.run.smithi028.stdout:34359738372 2024-09-17T23:46:06.903 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2024-09-17T23:46:07.177 INFO:teuthology.orchestra.run.smithi028.stdout:34359738371 2024-09-17T23:46:07.194 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.1 2024-09-17T23:46:07.259 INFO:teuthology.orchestra.run.smithi028.stdout:34359738371 2024-09-17T23:46:07.260 INFO:teuthology.orchestra.run.smithi028.stdout:34359738371 2024-09-17T23:46:07.279 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.2 2024-09-17T23:46:07.280 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.0 2024-09-17T23:46:08.195 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2024-09-17T23:46:08.280 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2024-09-17T23:46:08.281 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2024-09-17T23:46:08.538 INFO:teuthology.orchestra.run.smithi028.stdout:34359738372 2024-09-17T23:46:08.563 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.1 2024-09-17T23:46:08.563 DEBUG:teuthology.parallel:result is None 2024-09-17T23:46:08.575 INFO:teuthology.orchestra.run.smithi028.stdout:34359738372 2024-09-17T23:46:08.592 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.0 2024-09-17T23:46:08.592 DEBUG:teuthology.parallel:result is None 2024-09-17T23:46:08.624 INFO:teuthology.orchestra.run.smithi028.stdout:34359738372 2024-09-17T23:46:08.641 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.2 2024-09-17T23:46:08.641 DEBUG:teuthology.parallel:result is None 2024-09-17T23:46:08.641 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2024-09-17T23:46:08.641 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-17T23:46:08.973 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:08.973 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-09-17T23:46:08.989 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2024-09-17T23:46:07.688166+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":13,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":95120,"kb_used_data":1536,"kb_used_omap":4,"kb_used_meta":93499,"kb_avail":283020400,"statfs":{"total":289910292480,"available":289812889600,"internally_reserved":0,"allocated":1572864,"data_stored":1288919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":95743311},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.973161"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.734682+0000","last_change":"2024-09-17T23:46:05.734682+0000","last_active":"2024-09-17T23:46:05.734682+0000","last_peered":"2024-09-17T23:46:05.734682+0000","last_clean":"2024-09-17T23:46:05.734682+0000","last_became_active":"2024-09-17T23:46:03.734118+0000","last_became_peered":"2024-09-17T23:46:03.734118+0000","last_unstale":"2024-09-17T23:46:05.734682+0000","last_undegraded":"2024-09-17T23:46:05.734682+0000","last_fullsized":"2024-09-17T23:46:05.734682+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:58:19.460625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002359271,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.734204+0000","last_change":"2024-09-17T23:46:05.734204+0000","last_active":"2024-09-17T23:46:05.734204+0000","last_peered":"2024-09-17T23:46:05.734204+0000","last_clean":"2024-09-17T23:46:05.734204+0000","last_became_active":"2024-09-17T23:46:03.734798+0000","last_became_peered":"2024-09-17T23:46:03.734798+0000","last_unstale":"2024-09-17T23:46:05.734204+0000","last_undegraded":"2024-09-17T23:46:05.734204+0000","last_fullsized":"2024-09-17T23:46:05.734204+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T00:38:24.564262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018543850000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.734026+0000","last_change":"2024-09-17T23:46:05.734026+0000","last_active":"2024-09-17T23:46:05.734026+0000","last_peered":"2024-09-17T23:46:05.734026+0000","last_clean":"2024-09-17T23:46:05.734026+0000","last_became_active":"2024-09-17T23:46:03.733171+0000","last_became_peered":"2024-09-17T23:46:03.733171+0000","last_unstale":"2024-09-17T23:46:05.734026+0000","last_undegraded":"2024-09-17T23:46:05.734026+0000","last_fullsized":"2024-09-17T23:46:05.734026+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:02:46.037354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016983600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.733455+0000","last_change":"2024-09-17T23:46:05.733455+0000","last_active":"2024-09-17T23:46:05.733455+0000","last_peered":"2024-09-17T23:46:05.733455+0000","last_clean":"2024-09-17T23:46:05.733455+0000","last_became_active":"2024-09-17T23:46:03.733266+0000","last_became_peered":"2024-09-17T23:46:03.733266+0000","last_unstale":"2024-09-17T23:46:05.733455+0000","last_undegraded":"2024-09-17T23:46:05.733455+0000","last_fullsized":"2024-09-17T23:46:05.733455+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:46:17.826212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001165857,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.739829+0000","last_change":"2024-09-17T23:46:05.739829+0000","last_active":"2024-09-17T23:46:05.739829+0000","last_peered":"2024-09-17T23:46:05.739829+0000","last_clean":"2024-09-17T23:46:05.739829+0000","last_became_active":"2024-09-17T23:46:03.738379+0000","last_became_peered":"2024-09-17T23:46:03.738379+0000","last_unstale":"2024-09-17T23:46:05.739829+0000","last_undegraded":"2024-09-17T23:46:05.739829+0000","last_fullsized":"2024-09-17T23:46:05.739829+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:26:27.164769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019871049999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:06.244182+0000","last_change":"2024-09-17T23:46:06.244182+0000","last_active":"2024-09-17T23:46:06.244182+0000","last_peered":"2024-09-17T23:46:06.244182+0000","last_clean":"2024-09-17T23:46:06.244182+0000","last_became_active":"2024-09-17T23:46:03.730771+0000","last_became_peered":"2024-09-17T23:46:03.730771+0000","last_unstale":"2024-09-17T23:46:06.244182+0000","last_undegraded":"2024-09-17T23:46:06.244182+0000","last_fullsized":"2024-09-17T23:46:06.244182+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:32:53.624364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040605900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:06.243995+0000","last_change":"2024-09-17T23:46:06.243995+0000","last_active":"2024-09-17T23:46:06.243995+0000","last_peered":"2024-09-17T23:46:06.243995+0000","last_clean":"2024-09-17T23:46:06.243995+0000","last_became_active":"2024-09-17T23:46:03.731476+0000","last_became_peered":"2024-09-17T23:46:03.731476+0000","last_unstale":"2024-09-17T23:46:06.243995+0000","last_undegraded":"2024-09-17T23:46:06.243995+0000","last_fullsized":"2024-09-17T23:46:06.243995+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:54:18.516230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031242999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.733399+0000","last_change":"2024-09-17T23:46:05.733399+0000","last_active":"2024-09-17T23:46:05.733399+0000","last_peered":"2024-09-17T23:46:05.733399+0000","last_clean":"2024-09-17T23:46:05.733399+0000","last_became_active":"2024-09-17T23:46:03.729614+0000","last_became_peered":"2024-09-17T23:46:03.729614+0000","last_unstale":"2024-09-17T23:46:05.733399+0000","last_undegraded":"2024-09-17T23:46:05.733399+0000","last_fullsized":"2024-09-17T23:46:05.733399+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:19:46.872257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011240410000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'97","reported_seq":147,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.731880+0000","last_change":"2024-09-17T23:45:59.372129+0000","last_active":"2024-09-17T23:46:05.731880+0000","last_peered":"2024-09-17T23:46:05.731880+0000","last_clean":"2024-09-17T23:46:05.731880+0000","last_became_active":"2024-09-17T23:45:59.369113+0000","last_became_peered":"2024-09-17T23:45:59.369113+0000","last_unstale":"2024-09-17T23:46:05.731880+0000","last_undegraded":"2024-09-17T23:46:05.731880+0000","last_fullsized":"2024-09-17T23:46:05.731880+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:45:58.030243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:45:58.030243+0000","last_clean_scrub_stamp":"2024-09-17T23:45:58.030243+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:08:20.028364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31304,"kb_used_data":112,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340536,"statfs":{"total":96636764160,"available":96604708864,"internally_reserved":0,"allocated":114688,"data_stored":31355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31908,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339932,"statfs":{"total":96636764160,"available":96604090368,"internally_reserved":0,"allocated":729088,"data_stored":628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31908,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339932,"statfs":{"total":96636764160,"available":96604090368,"internally_reserved":0,"allocated":729088,"data_stored":628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T23:46:08.990 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-17T23:46:09.289 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:09.290 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-09-17T23:46:09.308 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2024-09-17T23:46:07.688166+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":13,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":95120,"kb_used_data":1536,"kb_used_omap":4,"kb_used_meta":93499,"kb_avail":283020400,"statfs":{"total":289910292480,"available":289812889600,"internally_reserved":0,"allocated":1572864,"data_stored":1288919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":95743311},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.973161"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.734682+0000","last_change":"2024-09-17T23:46:05.734682+0000","last_active":"2024-09-17T23:46:05.734682+0000","last_peered":"2024-09-17T23:46:05.734682+0000","last_clean":"2024-09-17T23:46:05.734682+0000","last_became_active":"2024-09-17T23:46:03.734118+0000","last_became_peered":"2024-09-17T23:46:03.734118+0000","last_unstale":"2024-09-17T23:46:05.734682+0000","last_undegraded":"2024-09-17T23:46:05.734682+0000","last_fullsized":"2024-09-17T23:46:05.734682+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:58:19.460625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002359271,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.734204+0000","last_change":"2024-09-17T23:46:05.734204+0000","last_active":"2024-09-17T23:46:05.734204+0000","last_peered":"2024-09-17T23:46:05.734204+0000","last_clean":"2024-09-17T23:46:05.734204+0000","last_became_active":"2024-09-17T23:46:03.734798+0000","last_became_peered":"2024-09-17T23:46:03.734798+0000","last_unstale":"2024-09-17T23:46:05.734204+0000","last_undegraded":"2024-09-17T23:46:05.734204+0000","last_fullsized":"2024-09-17T23:46:05.734204+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T00:38:24.564262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018543850000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.734026+0000","last_change":"2024-09-17T23:46:05.734026+0000","last_active":"2024-09-17T23:46:05.734026+0000","last_peered":"2024-09-17T23:46:05.734026+0000","last_clean":"2024-09-17T23:46:05.734026+0000","last_became_active":"2024-09-17T23:46:03.733171+0000","last_became_peered":"2024-09-17T23:46:03.733171+0000","last_unstale":"2024-09-17T23:46:05.734026+0000","last_undegraded":"2024-09-17T23:46:05.734026+0000","last_fullsized":"2024-09-17T23:46:05.734026+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:02:46.037354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016983600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.733455+0000","last_change":"2024-09-17T23:46:05.733455+0000","last_active":"2024-09-17T23:46:05.733455+0000","last_peered":"2024-09-17T23:46:05.733455+0000","last_clean":"2024-09-17T23:46:05.733455+0000","last_became_active":"2024-09-17T23:46:03.733266+0000","last_became_peered":"2024-09-17T23:46:03.733266+0000","last_unstale":"2024-09-17T23:46:05.733455+0000","last_undegraded":"2024-09-17T23:46:05.733455+0000","last_fullsized":"2024-09-17T23:46:05.733455+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:46:17.826212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001165857,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.739829+0000","last_change":"2024-09-17T23:46:05.739829+0000","last_active":"2024-09-17T23:46:05.739829+0000","last_peered":"2024-09-17T23:46:05.739829+0000","last_clean":"2024-09-17T23:46:05.739829+0000","last_became_active":"2024-09-17T23:46:03.738379+0000","last_became_peered":"2024-09-17T23:46:03.738379+0000","last_unstale":"2024-09-17T23:46:05.739829+0000","last_undegraded":"2024-09-17T23:46:05.739829+0000","last_fullsized":"2024-09-17T23:46:05.739829+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:26:27.164769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019871049999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:06.244182+0000","last_change":"2024-09-17T23:46:06.244182+0000","last_active":"2024-09-17T23:46:06.244182+0000","last_peered":"2024-09-17T23:46:06.244182+0000","last_clean":"2024-09-17T23:46:06.244182+0000","last_became_active":"2024-09-17T23:46:03.730771+0000","last_became_peered":"2024-09-17T23:46:03.730771+0000","last_unstale":"2024-09-17T23:46:06.244182+0000","last_undegraded":"2024-09-17T23:46:06.244182+0000","last_fullsized":"2024-09-17T23:46:06.244182+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:32:53.624364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040605900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:06.243995+0000","last_change":"2024-09-17T23:46:06.243995+0000","last_active":"2024-09-17T23:46:06.243995+0000","last_peered":"2024-09-17T23:46:06.243995+0000","last_clean":"2024-09-17T23:46:06.243995+0000","last_became_active":"2024-09-17T23:46:03.731476+0000","last_became_peered":"2024-09-17T23:46:03.731476+0000","last_unstale":"2024-09-17T23:46:06.243995+0000","last_undegraded":"2024-09-17T23:46:06.243995+0000","last_fullsized":"2024-09-17T23:46:06.243995+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:54:18.516230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031242999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.733399+0000","last_change":"2024-09-17T23:46:05.733399+0000","last_active":"2024-09-17T23:46:05.733399+0000","last_peered":"2024-09-17T23:46:05.733399+0000","last_clean":"2024-09-17T23:46:05.733399+0000","last_became_active":"2024-09-17T23:46:03.729614+0000","last_became_peered":"2024-09-17T23:46:03.729614+0000","last_unstale":"2024-09-17T23:46:05.733399+0000","last_undegraded":"2024-09-17T23:46:05.733399+0000","last_fullsized":"2024-09-17T23:46:05.733399+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:19:46.872257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011240410000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'97","reported_seq":147,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-09-17T23:46:05.731880+0000","last_change":"2024-09-17T23:45:59.372129+0000","last_active":"2024-09-17T23:46:05.731880+0000","last_peered":"2024-09-17T23:46:05.731880+0000","last_clean":"2024-09-17T23:46:05.731880+0000","last_became_active":"2024-09-17T23:45:59.369113+0000","last_became_peered":"2024-09-17T23:45:59.369113+0000","last_unstale":"2024-09-17T23:46:05.731880+0000","last_undegraded":"2024-09-17T23:46:05.731880+0000","last_fullsized":"2024-09-17T23:46:05.731880+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:45:58.030243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:45:58.030243+0000","last_clean_scrub_stamp":"2024-09-17T23:45:58.030243+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:08:20.028364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31304,"kb_used_data":112,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94340536,"statfs":{"total":96636764160,"available":96604708864,"internally_reserved":0,"allocated":114688,"data_stored":31355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31908,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339932,"statfs":{"total":96636764160,"available":96604090368,"internally_reserved":0,"allocated":729088,"data_stored":628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31908,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339932,"statfs":{"total":96636764160,"available":96604090368,"internally_reserved":0,"allocated":729088,"data_stored":628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T23:46:09.309 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2024-09-17T23:46:09.309 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2024-09-17T23:46:09.309 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2024-09-17T23:46:09.309 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2024-09-17T23:46:09.649 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:09.649 INFO:teuthology.orchestra.run.smithi028.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-17T23:46:09.668 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2024-09-17T23:46:09.668 INFO:teuthology.run_tasks:Running task ceph... 2024-09-17T23:46:09.680 INFO:tasks.ceph:Creating ceph cluster cluster2... 2024-09-17T23:46:09.680 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-09-17T23:46:09.681 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601', 'branch': 'squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid supported-random-distro$/{ubuntu_latest} workloads/compare-mirror-images-nbd}', 'email': 'ceph-qa@ceph.com', 'first_in_suite': False, 'job_id': '7909601', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}], 'name': 'teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'extra_packages': ['rbd-nbd'], 'flavor': 'default', 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}, 'extra_system_packages': ['pv']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'squid', 'sha1': 'b0d356b98e08f6bc59ef15652f0d0e4298fc83c7'}}, 'owner': 'scheduled_teuthology@teuthology', 'priority': 100, 'repo': 'https://git.ceph.com/ceph.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 1983, 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f', 'sleep_before_teardown': 0, 'subset': '79/128', 'suite': 'rbd', 'suite_branch': 'squid', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph_b0d356b98e08f6bc59ef15652f0d0e4298fc83c7/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph.git', 'suite_sha1': 'b0d356b98e08f6bc59ef15652f0d0e4298fc83c7', 'targets': {'smithi028.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJ6R7D/dpH5Q/kJUJlgC8va/DNhJaP+PmCcJJPaHFjdILYt4sac7Kdqus1jDVzYR5Dp/oPvfM3MhfFS26p5pLlo=', 'smithi049.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKHINc9zC/b1hH/hNMeqnZWqar4lztbQ1kZBO25TL4fCb59Evgo2aYxq63Ywrhh+do4yrTzxrlD+89bwhBYec8o='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror', 'rbd-nbd'], 'flavor': 'default', 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f', 'extra_system_packages': ['pv']}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/compare_mirror_images.sh']}, 'env': {'RBD_DEVICE_TYPE': 'nbd'}, 'timeout': '3h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-09-17_21:16:02', 'tube': 'smithi', 'user': 'teuthology', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856'} 2024-09-17T23:46:09.681 DEBUG:teuthology.orchestra.run.smithi028:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2024-09-17T23:46:09.684 DEBUG:teuthology.orchestra.run.smithi049:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2024-09-17T23:46:09.690 DEBUG:teuthology.orchestra.run.smithi028:> sudo install -d -m0777 -- /var/run/ceph 2024-09-17T23:46:09.732 DEBUG:teuthology.orchestra.run.smithi049:> sudo install -d -m0777 -- /var/run/ceph 2024-09-17T23:46:09.746 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:09.746 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T23:46:09.793 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T23:46:09.793 DEBUG:teuthology.orchestra.run.smithi049:> stat /dev/vg_nvme/lv_1 2024-09-17T23:46:09.841 INFO:teuthology.orchestra.run.smithi049.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T23:46:09.841 INFO:teuthology.orchestra.run.smithi049.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T23:46:09.841 INFO:teuthology.orchestra.run.smithi049.stdout:Device: 5h/5d Inode: 885 Links: 1 2024-09-17T23:46:09.841 INFO:teuthology.orchestra.run.smithi049.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T23:46:09.841 INFO:teuthology.orchestra.run.smithi049.stdout:Access: 2024-09-17 23:39:58.113773255 +0000 2024-09-17T23:46:09.841 INFO:teuthology.orchestra.run.smithi049.stdout:Modify: 2024-09-17 23:39:57.873775400 +0000 2024-09-17T23:46:09.841 INFO:teuthology.orchestra.run.smithi049.stdout:Change: 2024-09-17 23:39:57.873775400 +0000 2024-09-17T23:46:09.841 INFO:teuthology.orchestra.run.smithi049.stdout: Birth: - 2024-09-17T23:46:09.841 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T23:46:09.895 INFO:teuthology.orchestra.run.smithi049.stderr:1+0 records in 2024-09-17T23:46:09.895 INFO:teuthology.orchestra.run.smithi049.stderr:1+0 records out 2024-09-17T23:46:09.895 INFO:teuthology.orchestra.run.smithi049.stderr:512 bytes copied, 0.000299094 s, 1.7 MB/s 2024-09-17T23:46:09.896 DEBUG:teuthology.orchestra.run.smithi049:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T23:46:09.946 DEBUG:teuthology.orchestra.run.smithi049:> stat /dev/vg_nvme/lv_2 2024-09-17T23:46:09.993 INFO:teuthology.orchestra.run.smithi049.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T23:46:09.993 INFO:teuthology.orchestra.run.smithi049.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T23:46:09.993 INFO:teuthology.orchestra.run.smithi049.stdout:Device: 5h/5d Inode: 897 Links: 1 2024-09-17T23:46:09.993 INFO:teuthology.orchestra.run.smithi049.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T23:46:09.994 INFO:teuthology.orchestra.run.smithi049.stdout:Access: 2024-09-17 23:39:58.605768862 +0000 2024-09-17T23:46:09.994 INFO:teuthology.orchestra.run.smithi049.stdout:Modify: 2024-09-17 23:39:58.349771147 +0000 2024-09-17T23:46:09.994 INFO:teuthology.orchestra.run.smithi049.stdout:Change: 2024-09-17 23:39:58.349771147 +0000 2024-09-17T23:46:09.994 INFO:teuthology.orchestra.run.smithi049.stdout: Birth: - 2024-09-17T23:46:09.994 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T23:46:10.047 INFO:teuthology.orchestra.run.smithi049.stderr:1+0 records in 2024-09-17T23:46:10.047 INFO:teuthology.orchestra.run.smithi049.stderr:1+0 records out 2024-09-17T23:46:10.047 INFO:teuthology.orchestra.run.smithi049.stderr:512 bytes copied, 0.000296335 s, 1.7 MB/s 2024-09-17T23:46:10.049 DEBUG:teuthology.orchestra.run.smithi049:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T23:46:10.098 DEBUG:teuthology.orchestra.run.smithi049:> stat /dev/vg_nvme/lv_3 2024-09-17T23:46:10.145 INFO:teuthology.orchestra.run.smithi049.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T23:46:10.145 INFO:teuthology.orchestra.run.smithi049.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T23:46:10.145 INFO:teuthology.orchestra.run.smithi049.stdout:Device: 5h/5d Inode: 917 Links: 1 2024-09-17T23:46:10.145 INFO:teuthology.orchestra.run.smithi049.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T23:46:10.146 INFO:teuthology.orchestra.run.smithi049.stdout:Access: 2024-09-17 23:39:59.105764395 +0000 2024-09-17T23:46:10.146 INFO:teuthology.orchestra.run.smithi049.stdout:Modify: 2024-09-17 23:39:58.845766717 +0000 2024-09-17T23:46:10.146 INFO:teuthology.orchestra.run.smithi049.stdout:Change: 2024-09-17 23:39:58.845766717 +0000 2024-09-17T23:46:10.146 INFO:teuthology.orchestra.run.smithi049.stdout: Birth: - 2024-09-17T23:46:10.146 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T23:46:10.199 INFO:teuthology.orchestra.run.smithi049.stderr:1+0 records in 2024-09-17T23:46:10.200 INFO:teuthology.orchestra.run.smithi049.stderr:1+0 records out 2024-09-17T23:46:10.200 INFO:teuthology.orchestra.run.smithi049.stderr:512 bytes copied, 0.000293901 s, 1.7 MB/s 2024-09-17T23:46:10.201 DEBUG:teuthology.orchestra.run.smithi049:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T23:46:10.250 DEBUG:teuthology.orchestra.run.smithi049:> stat /dev/vg_nvme/lv_4 2024-09-17T23:46:10.297 INFO:teuthology.orchestra.run.smithi049.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T23:46:10.297 INFO:teuthology.orchestra.run.smithi049.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T23:46:10.297 INFO:teuthology.orchestra.run.smithi049.stdout:Device: 5h/5d Inode: 934 Links: 1 2024-09-17T23:46:10.297 INFO:teuthology.orchestra.run.smithi049.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T23:46:10.297 INFO:teuthology.orchestra.run.smithi049.stdout:Access: 2024-09-17 23:39:59.589760071 +0000 2024-09-17T23:46:10.297 INFO:teuthology.orchestra.run.smithi049.stdout:Modify: 2024-09-17 23:39:59.337762323 +0000 2024-09-17T23:46:10.297 INFO:teuthology.orchestra.run.smithi049.stdout:Change: 2024-09-17 23:39:59.337762323 +0000 2024-09-17T23:46:10.297 INFO:teuthology.orchestra.run.smithi049.stdout: Birth: - 2024-09-17T23:46:10.298 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T23:46:10.355 INFO:teuthology.orchestra.run.smithi049.stderr:1+0 records in 2024-09-17T23:46:10.355 INFO:teuthology.orchestra.run.smithi049.stderr:1+0 records out 2024-09-17T23:46:10.355 INFO:teuthology.orchestra.run.smithi049.stderr:512 bytes copied, 0.000258644 s, 2.0 MB/s 2024-09-17T23:46:10.356 DEBUG:teuthology.orchestra.run.smithi049:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T23:46:10.406 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-17T23:46:10.406 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi049.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-17T23:46:10.406 INFO:tasks.ceph:Generating config... 2024-09-17T23:46:10.407 INFO:tasks.ceph:[client] debug rbd = 20 2024-09-17T23:46:10.408 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2024-09-17T23:46:10.408 INFO:tasks.ceph:[client] log to stderr = False 2024-09-17T23:46:10.408 INFO:tasks.ceph:[client] rbd default features = 125 2024-09-17T23:46:10.408 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.408 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.408 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.408 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.408 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.408 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.408 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.408 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.408 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.409 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.409 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.409 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.409 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.409 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.409 INFO:tasks.ceph:[global] mon client directed command retry = 5 2024-09-17T23:46:10.409 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-09-17T23:46:10.409 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2024-09-17T23:46:10.409 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-09-17T23:46:10.409 INFO:tasks.ceph:[mgr] debug ms = 1 2024-09-17T23:46:10.409 INFO:tasks.ceph:[mon] debug mon = 20 2024-09-17T23:46:10.409 INFO:tasks.ceph:[mon] debug ms = 1 2024-09-17T23:46:10.409 INFO:tasks.ceph:[mon] debug paxos = 20 2024-09-17T23:46:10.409 INFO:tasks.ceph:[osd] bluefs allocator = hybrid 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] bluestore allocator = hybrid 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] debug ms = 1 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] debug osd = 20 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-09-17T23:46:10.410 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-09-17T23:46:10.411 INFO:tasks.ceph:Setting up cluster2.mon.a... 2024-09-17T23:46:10.411 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2024-09-17T23:46:10.486 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster2.keyring 2024-09-17T23:46:10.489 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2024-09-17T23:46:10.570 DEBUG:teuthology.orchestra.run.smithi049:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-09-17T23:46:10.624 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '172.21.15.49')] 2024-09-17T23:46:10.624 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.49', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2024-09-17T23:46:10.625 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:10.625 DEBUG:teuthology.orchestra.run.smithi049:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-09-17T23:46:10.673 DEBUG:teuthology.orchestra.run.smithi049:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.49 --print /home/ubuntu/cephtest/cluster2.monmap 2024-09-17T23:46:10.754 INFO:teuthology.orchestra.run.smithi049.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2024-09-17T23:46:10.754 INFO:teuthology.orchestra.run.smithi049.stdout:monmaptool: generated fsid 024a7a4f-3e37-4953-ac7d-ad33b26dedaa 2024-09-17T23:46:10.754 INFO:teuthology.orchestra.run.smithi049.stdout:setting min_mon_release = quincy 2024-09-17T23:46:10.754 INFO:teuthology.orchestra.run.smithi049.stdout:epoch 0 2024-09-17T23:46:10.754 INFO:teuthology.orchestra.run.smithi049.stdout:fsid 024a7a4f-3e37-4953-ac7d-ad33b26dedaa 2024-09-17T23:46:10.754 INFO:teuthology.orchestra.run.smithi049.stdout:last_changed 2024-09-17T23:46:10.754448+0000 2024-09-17T23:46:10.754 INFO:teuthology.orchestra.run.smithi049.stdout:created 2024-09-17T23:46:10.754448+0000 2024-09-17T23:46:10.754 INFO:teuthology.orchestra.run.smithi049.stdout:min_mon_release 17 (quincy) 2024-09-17T23:46:10.755 INFO:teuthology.orchestra.run.smithi049.stdout:election_strategy: 1 2024-09-17T23:46:10.755 INFO:teuthology.orchestra.run.smithi049.stdout:0: [v2:172.21.15.49:3300/0,v1:172.21.15.49:6789/0] mon.a 2024-09-17T23:46:10.755 INFO:teuthology.orchestra.run.smithi049.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2024-09-17T23:46:10.756 DEBUG:teuthology.orchestra.run.smithi049:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-09-17T23:46:10.805 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 024a7a4f-3e37-4953-ac7d-ad33b26dedaa... 2024-09-17T23:46:10.806 DEBUG:teuthology.orchestra.run.smithi028:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2024-09-17T23:46:10.809 DEBUG:teuthology.orchestra.run.smithi049:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2024-09-17T23:46:10.851 INFO:teuthology.orchestra.run.smithi028.stdout:[global] 2024-09-17T23:46:10.851 INFO:teuthology.orchestra.run.smithi028.stdout: chdir = "" 2024-09-17T23:46:10.851 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-17T23:46:10.851 INFO:teuthology.orchestra.run.smithi028.stdout: auth supported = cephx 2024-09-17T23:46:10.851 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.852 INFO:teuthology.orchestra.run.smithi028.stdout: filestore xattr use omap = true 2024-09-17T23:46:10.852 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.852 INFO:teuthology.orchestra.run.smithi028.stdout: mon clock drift allowed = 1.000 2024-09-17T23:46:10.852 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.852 INFO:teuthology.orchestra.run.smithi028.stdout: osd crush chooseleaf type = 0 2024-09-17T23:46:10.852 INFO:teuthology.orchestra.run.smithi028.stdout: auth debug = true 2024-09-17T23:46:10.852 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.852 INFO:teuthology.orchestra.run.smithi028.stdout: ms die on old message = true 2024-09-17T23:46:10.852 INFO:teuthology.orchestra.run.smithi028.stdout: ms die on bug = true 2024-09-17T23:46:10.852 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.852 INFO:teuthology.orchestra.run.smithi028.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-17T23:46:10.852 INFO:teuthology.orchestra.run.smithi028.stdout: mon pg warn max object skew = 0 2024-09-17T23:46:10.853 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.853 INFO:teuthology.orchestra.run.smithi028.stdout: # disable pg_autoscaler by default for new pools 2024-09-17T23:46:10.853 INFO:teuthology.orchestra.run.smithi028.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-17T23:46:10.853 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.853 INFO:teuthology.orchestra.run.smithi028.stdout: osd pool default size = 2 2024-09-17T23:46:10.853 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.853 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd allow primary affinity = true 2024-09-17T23:46:10.853 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd allow pg remap = true 2024-09-17T23:46:10.854 INFO:teuthology.orchestra.run.smithi028.stdout: mon warn on legacy crush tunables = false 2024-09-17T23:46:10.854 INFO:teuthology.orchestra.run.smithi028.stdout: mon warn on crush straw calc version zero = false 2024-09-17T23:46:10.854 INFO:teuthology.orchestra.run.smithi028.stdout: mon warn on no sortbitwise = false 2024-09-17T23:46:10.854 INFO:teuthology.orchestra.run.smithi028.stdout: mon warn on osd down out interval zero = false 2024-09-17T23:46:10.854 INFO:teuthology.orchestra.run.smithi028.stdout: mon warn on too few osds = false 2024-09-17T23:46:10.854 INFO:teuthology.orchestra.run.smithi028.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-17T23:46:10.854 INFO:teuthology.orchestra.run.smithi028.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-17T23:46:10.854 INFO:teuthology.orchestra.run.smithi028.stdout: mon_allow_pool_size_one = true 2024-09-17T23:46:10.854 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.854 INFO:teuthology.orchestra.run.smithi028.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-17T23:46:10.854 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.854 INFO:teuthology.orchestra.run.smithi028.stdout: osd default data pool replay window = 5 2024-09-17T23:46:10.855 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.855 INFO:teuthology.orchestra.run.smithi028.stdout: mon allow pool delete = true 2024-09-17T23:46:10.855 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.855 INFO:teuthology.orchestra.run.smithi028.stdout: mon cluster log file level = debug 2024-09-17T23:46:10.855 INFO:teuthology.orchestra.run.smithi028.stdout: debug asserts on shutdown = true 2024-09-17T23:46:10.855 INFO:teuthology.orchestra.run.smithi028.stdout: mon health detail to clog = false 2024-09-17T23:46:10.855 INFO:teuthology.orchestra.run.smithi028.stdout: mon host = 172.21.15.49 2024-09-17T23:46:10.855 INFO:teuthology.orchestra.run.smithi028.stdout: mon client directed command retry = 5 2024-09-17T23:46:10.855 INFO:teuthology.orchestra.run.smithi028.stdout: mon warn on pool no app = False 2024-09-17T23:46:10.855 INFO:teuthology.orchestra.run.smithi028.stdout: ms inject socket failures = 5000 2024-09-17T23:46:10.855 INFO:teuthology.orchestra.run.smithi028.stdout: fsid = 024a7a4f-3e37-4953-ac7d-ad33b26dedaa 2024-09-17T23:46:10.855 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout:[osd] 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout: osd journal size = 100 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout: osd scrub load threshold = 5.0 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout: osd scrub max interval = 600 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout: osd mclock profile = high_recovery_ops 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout: osd recover clone overlap = true 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout: osd recovery max chunk = 1048576 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout: osd debug shutdown = true 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout: osd debug op order = true 2024-09-17T23:46:10.856 INFO:teuthology.orchestra.run.smithi028.stdout: osd debug verify stray on activate = true 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: osd debug trim objects = true 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: osd open classes on start = true 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: osd debug pg log writeout = true 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: osd deep scrub update digest min age = 30 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: osd map max advance = 10 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: journal zero on create = true 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: filestore ondisk finisher threads = 3 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: filestore apply finisher threads = 3 2024-09-17T23:46:10.857 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.858 INFO:teuthology.orchestra.run.smithi028.stdout: bdev debug aio = true 2024-09-17T23:46:10.858 INFO:teuthology.orchestra.run.smithi028.stdout: osd debug misdirected ops = true 2024-09-17T23:46:10.858 INFO:teuthology.orchestra.run.smithi028.stdout: bluefs allocator = hybrid 2024-09-17T23:46:10.858 INFO:teuthology.orchestra.run.smithi028.stdout: bluestore allocator = hybrid 2024-09-17T23:46:10.858 INFO:teuthology.orchestra.run.smithi028.stdout: bluestore block size = 96636764160 2024-09-17T23:46:10.858 INFO:teuthology.orchestra.run.smithi028.stdout: bluestore fsck on mount = True 2024-09-17T23:46:10.858 INFO:teuthology.orchestra.run.smithi028.stdout: debug bluefs = 1/20 2024-09-17T23:46:10.858 INFO:teuthology.orchestra.run.smithi028.stdout: debug bluestore = 1/20 2024-09-17T23:46:10.858 INFO:teuthology.orchestra.run.smithi028.stdout: debug ms = 1 2024-09-17T23:46:10.858 INFO:teuthology.orchestra.run.smithi028.stdout: debug osd = 20 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: debug rocksdb = 4/10 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd backfillfull_ratio = 0.85 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd full ratio = 0.9 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd nearfull ratio = 0.8 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: osd failsafe full ratio = 0.95 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: osd objectstore = bluestore 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout:[mgr] 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: debug ms = 1 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: debug mgr = 20 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: debug mon = 20 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: debug auth = 20 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: mon reweight min pgs per osd = 4 2024-09-17T23:46:10.859 INFO:teuthology.orchestra.run.smithi028.stdout: mon reweight min bytes per osd = 10 2024-09-17T23:46:10.860 INFO:teuthology.orchestra.run.smithi028.stdout: mgr/telemetry/nag = false 2024-09-17T23:46:10.860 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.860 INFO:teuthology.orchestra.run.smithi028.stdout:[mon] 2024-09-17T23:46:10.860 INFO:teuthology.orchestra.run.smithi028.stdout: debug ms = 1 2024-09-17T23:46:10.860 INFO:teuthology.orchestra.run.smithi028.stdout: debug mon = 20 2024-09-17T23:46:10.860 INFO:teuthology.orchestra.run.smithi028.stdout: debug paxos = 20 2024-09-17T23:46:10.860 INFO:teuthology.orchestra.run.smithi028.stdout: debug auth = 20 2024-09-17T23:46:10.860 INFO:teuthology.orchestra.run.smithi028.stdout: mon data avail warn = 5 2024-09-17T23:46:10.860 INFO:teuthology.orchestra.run.smithi028.stdout: mon mgr mkfs grace = 240 2024-09-17T23:46:10.860 INFO:teuthology.orchestra.run.smithi028.stdout: mon reweight min pgs per osd = 4 2024-09-17T23:46:10.860 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd reporter subtree level = osd 2024-09-17T23:46:10.861 INFO:teuthology.orchestra.run.smithi028.stdout: mon osd prime pg temp = true 2024-09-17T23:46:10.861 INFO:teuthology.orchestra.run.smithi028.stdout: mon reweight min bytes per osd = 10 2024-09-17T23:46:10.861 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.861 INFO:teuthology.orchestra.run.smithi028.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-17T23:46:10.861 INFO:teuthology.orchestra.run.smithi028.stdout: auth mon ticket ttl = 660 # 11m 2024-09-17T23:46:10.861 INFO:teuthology.orchestra.run.smithi028.stdout: auth service ticket ttl = 240 # 4m 2024-09-17T23:46:10.861 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.861 INFO:teuthology.orchestra.run.smithi028.stdout: # don't complain about insecure global_id in the test suite 2024-09-17T23:46:10.861 INFO:teuthology.orchestra.run.smithi028.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-17T23:46:10.861 INFO:teuthology.orchestra.run.smithi028.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-17T23:46:10.861 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: # 1m isn't quite enough 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: mon_down_mkfs_grace = 2m 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: mon_warn_on_filestore_osds = false 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout:[client] 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: rgw cache enabled = true 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: rgw enable ops log = true 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: rgw enable usage log = true 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: debug rbd = 20 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: debug rbd_mirror = 15 2024-09-17T23:46:10.862 INFO:teuthology.orchestra.run.smithi028.stdout: log to stderr = False 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout: rbd default features = 125 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout:[mon.a] 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.0] 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.1] 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.2] 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.3] 2024-09-17T23:46:10.863 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.864 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.864 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.4] 2024-09-17T23:46:10.864 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.864 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.864 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.5] 2024-09-17T23:46:10.864 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.864 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.864 INFO:teuthology.orchestra.run.smithi028.stdout:[client.mirror.6] 2024-09-17T23:46:10.864 INFO:teuthology.orchestra.run.smithi028.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.864 INFO:teuthology.orchestra.run.smithi028.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.879 INFO:teuthology.orchestra.run.smithi049.stdout:[global] 2024-09-17T23:46:10.880 INFO:teuthology.orchestra.run.smithi049.stdout: chdir = "" 2024-09-17T23:46:10.880 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-17T23:46:10.880 INFO:teuthology.orchestra.run.smithi049.stdout: auth supported = cephx 2024-09-17T23:46:10.880 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.880 INFO:teuthology.orchestra.run.smithi049.stdout: filestore xattr use omap = true 2024-09-17T23:46:10.881 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.881 INFO:teuthology.orchestra.run.smithi049.stdout: mon clock drift allowed = 1.000 2024-09-17T23:46:10.881 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.881 INFO:teuthology.orchestra.run.smithi049.stdout: osd crush chooseleaf type = 0 2024-09-17T23:46:10.881 INFO:teuthology.orchestra.run.smithi049.stdout: auth debug = true 2024-09-17T23:46:10.881 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.882 INFO:teuthology.orchestra.run.smithi049.stdout: ms die on old message = true 2024-09-17T23:46:10.882 INFO:teuthology.orchestra.run.smithi049.stdout: ms die on bug = true 2024-09-17T23:46:10.882 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.882 INFO:teuthology.orchestra.run.smithi049.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-17T23:46:10.882 INFO:teuthology.orchestra.run.smithi049.stdout: mon pg warn max object skew = 0 2024-09-17T23:46:10.883 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.883 INFO:teuthology.orchestra.run.smithi049.stdout: # disable pg_autoscaler by default for new pools 2024-09-17T23:46:10.883 INFO:teuthology.orchestra.run.smithi049.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-17T23:46:10.883 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.883 INFO:teuthology.orchestra.run.smithi049.stdout: osd pool default size = 2 2024-09-17T23:46:10.883 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.884 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd allow primary affinity = true 2024-09-17T23:46:10.884 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd allow pg remap = true 2024-09-17T23:46:10.884 INFO:teuthology.orchestra.run.smithi049.stdout: mon warn on legacy crush tunables = false 2024-09-17T23:46:10.884 INFO:teuthology.orchestra.run.smithi049.stdout: mon warn on crush straw calc version zero = false 2024-09-17T23:46:10.884 INFO:teuthology.orchestra.run.smithi049.stdout: mon warn on no sortbitwise = false 2024-09-17T23:46:10.885 INFO:teuthology.orchestra.run.smithi049.stdout: mon warn on osd down out interval zero = false 2024-09-17T23:46:10.885 INFO:teuthology.orchestra.run.smithi049.stdout: mon warn on too few osds = false 2024-09-17T23:46:10.885 INFO:teuthology.orchestra.run.smithi049.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-17T23:46:10.885 INFO:teuthology.orchestra.run.smithi049.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-17T23:46:10.885 INFO:teuthology.orchestra.run.smithi049.stdout: mon_allow_pool_size_one = true 2024-09-17T23:46:10.885 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.886 INFO:teuthology.orchestra.run.smithi049.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-17T23:46:10.886 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.886 INFO:teuthology.orchestra.run.smithi049.stdout: osd default data pool replay window = 5 2024-09-17T23:46:10.886 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.886 INFO:teuthology.orchestra.run.smithi049.stdout: mon allow pool delete = true 2024-09-17T23:46:10.886 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.886 INFO:teuthology.orchestra.run.smithi049.stdout: mon cluster log file level = debug 2024-09-17T23:46:10.886 INFO:teuthology.orchestra.run.smithi049.stdout: debug asserts on shutdown = true 2024-09-17T23:46:10.887 INFO:teuthology.orchestra.run.smithi049.stdout: mon health detail to clog = false 2024-09-17T23:46:10.887 INFO:teuthology.orchestra.run.smithi049.stdout: mon host = 172.21.15.49 2024-09-17T23:46:10.887 INFO:teuthology.orchestra.run.smithi049.stdout: mon client directed command retry = 5 2024-09-17T23:46:10.887 INFO:teuthology.orchestra.run.smithi049.stdout: mon warn on pool no app = False 2024-09-17T23:46:10.887 INFO:teuthology.orchestra.run.smithi049.stdout: ms inject socket failures = 5000 2024-09-17T23:46:10.888 INFO:teuthology.orchestra.run.smithi049.stdout: fsid = 024a7a4f-3e37-4953-ac7d-ad33b26dedaa 2024-09-17T23:46:10.888 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.888 INFO:teuthology.orchestra.run.smithi049.stdout:[osd] 2024-09-17T23:46:10.888 INFO:teuthology.orchestra.run.smithi049.stdout: osd journal size = 100 2024-09-17T23:46:10.888 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.888 INFO:teuthology.orchestra.run.smithi049.stdout: osd scrub load threshold = 5.0 2024-09-17T23:46:10.888 INFO:teuthology.orchestra.run.smithi049.stdout: osd scrub max interval = 600 2024-09-17T23:46:10.888 INFO:teuthology.orchestra.run.smithi049.stdout: osd mclock profile = high_recovery_ops 2024-09-17T23:46:10.889 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.889 INFO:teuthology.orchestra.run.smithi049.stdout: osd recover clone overlap = true 2024-09-17T23:46:10.889 INFO:teuthology.orchestra.run.smithi049.stdout: osd recovery max chunk = 1048576 2024-09-17T23:46:10.889 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.889 INFO:teuthology.orchestra.run.smithi049.stdout: osd debug shutdown = true 2024-09-17T23:46:10.889 INFO:teuthology.orchestra.run.smithi049.stdout: osd debug op order = true 2024-09-17T23:46:10.889 INFO:teuthology.orchestra.run.smithi049.stdout: osd debug verify stray on activate = true 2024-09-17T23:46:10.889 INFO:teuthology.orchestra.run.smithi049.stdout: osd debug trim objects = true 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: osd open classes on start = true 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: osd debug pg log writeout = true 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: osd deep scrub update digest min age = 30 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: osd map max advance = 10 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: journal zero on create = true 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: filestore ondisk finisher threads = 3 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: filestore apply finisher threads = 3 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.890 INFO:teuthology.orchestra.run.smithi049.stdout: bdev debug aio = true 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: osd debug misdirected ops = true 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: bluefs allocator = hybrid 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: bluestore allocator = hybrid 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: bluestore block size = 96636764160 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: bluestore fsck on mount = True 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: debug bluefs = 1/20 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: debug bluestore = 1/20 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: debug ms = 1 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: debug osd = 20 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: debug rocksdb = 4/10 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd backfillfull_ratio = 0.85 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd full ratio = 0.9 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd nearfull ratio = 0.8 2024-09-17T23:46:10.891 INFO:teuthology.orchestra.run.smithi049.stdout: osd failsafe full ratio = 0.95 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout: osd objectstore = bluestore 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout:[mgr] 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout: debug ms = 1 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout: debug mgr = 20 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout: debug mon = 20 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout: debug auth = 20 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout: mon reweight min pgs per osd = 4 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout: mon reweight min bytes per osd = 10 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout: mgr/telemetry/nag = false 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout:[mon] 2024-09-17T23:46:10.892 INFO:teuthology.orchestra.run.smithi049.stdout: debug ms = 1 2024-09-17T23:46:10.893 INFO:teuthology.orchestra.run.smithi049.stdout: debug mon = 20 2024-09-17T23:46:10.893 INFO:teuthology.orchestra.run.smithi049.stdout: debug paxos = 20 2024-09-17T23:46:10.893 INFO:teuthology.orchestra.run.smithi049.stdout: debug auth = 20 2024-09-17T23:46:10.893 INFO:teuthology.orchestra.run.smithi049.stdout: mon data avail warn = 5 2024-09-17T23:46:10.893 INFO:teuthology.orchestra.run.smithi049.stdout: mon mgr mkfs grace = 240 2024-09-17T23:46:10.893 INFO:teuthology.orchestra.run.smithi049.stdout: mon reweight min pgs per osd = 4 2024-09-17T23:46:10.893 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd reporter subtree level = osd 2024-09-17T23:46:10.893 INFO:teuthology.orchestra.run.smithi049.stdout: mon osd prime pg temp = true 2024-09-17T23:46:10.893 INFO:teuthology.orchestra.run.smithi049.stdout: mon reweight min bytes per osd = 10 2024-09-17T23:46:10.893 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.893 INFO:teuthology.orchestra.run.smithi049.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout: auth mon ticket ttl = 660 # 11m 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout: auth service ticket ttl = 240 # 4m 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout: # don't complain about insecure global_id in the test suite 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout: # 1m isn't quite enough 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout: mon_down_mkfs_grace = 2m 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout: mon_warn_on_filestore_osds = false 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:10.894 INFO:teuthology.orchestra.run.smithi049.stdout:[client] 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout: rgw cache enabled = true 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout: rgw enable ops log = true 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout: rgw enable usage log = true 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout: debug rbd = 20 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout: debug rbd_mirror = 15 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout: log to stderr = False 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout: rbd default features = 125 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout:[mon.a] 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.0] 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.895 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.1] 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.2] 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.3] 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.4] 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.5] 2024-09-17T23:46:10.896 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.897 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.897 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.6] 2024-09-17T23:46:10.897 INFO:teuthology.orchestra.run.smithi049.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-17T23:46:10.897 INFO:teuthology.orchestra.run.smithi049.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-17T23:46:10.898 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2024-09-17T23:46:10.898 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2024-09-17T23:46:10.996 INFO:tasks.ceph:Copying monmap to all nodes... 2024-09-17T23:46:10.996 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:10.996 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2024-09-17T23:46:11.003 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:11.003 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2024-09-17T23:46:11.048 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi028.front.sepia.ceph.com 2024-09-17T23:46:11.048 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:46:11.049 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd of=/etc/ceph/cluster2.keyring 2024-09-17T23:46:11.049 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-09-17T23:46:11.073 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:46:11.073 DEBUG:teuthology.orchestra.run.smithi028:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2024-09-17T23:46:11.120 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi049.front.sepia.ceph.com 2024-09-17T23:46:11.120 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:11.120 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd of=/etc/ceph/cluster2.keyring 2024-09-17T23:46:11.120 DEBUG:teuthology.orchestra.run.smithi049:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-09-17T23:46:11.145 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:11.145 DEBUG:teuthology.orchestra.run.smithi049:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2024-09-17T23:46:11.193 INFO:tasks.ceph:Setting up mon nodes... 2024-09-17T23:46:11.193 INFO:tasks.ceph:Setting up mgr nodes... 2024-09-17T23:46:11.194 DEBUG:teuthology.orchestra.run.smithi028:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2024-09-17T23:46:11.238 INFO:teuthology.orchestra.run.smithi028.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2024-09-17T23:46:11.241 INFO:tasks.ceph:Setting up mds nodes... 2024-09-17T23:46:11.242 INFO:tasks.ceph_client:Setting up client nodes... 2024-09-17T23:46:11.242 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2024-09-17T23:46:11.318 INFO:teuthology.orchestra.run.smithi028.stdout:creating /etc/ceph/cluster2.client.0.keyring 2024-09-17T23:46:11.330 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2024-09-17T23:46:11.366 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2024-09-17T23:46:11.377 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2024-09-17T23:46:11.455 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2024-09-17T23:46:11.466 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2024-09-17T23:46:11.541 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2024-09-17T23:46:11.552 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2024-09-17T23:46:11.629 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2024-09-17T23:46:11.640 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2024-09-17T23:46:11.721 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2024-09-17T23:46:11.733 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2024-09-17T23:46:11.812 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2024-09-17T23:46:11.823 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2024-09-17T23:46:11.896 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2024-09-17T23:46:11.907 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2024-09-17T23:46:11.984 INFO:teuthology.orchestra.run.smithi049.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2024-09-17T23:46:11.994 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-09-17T23:46:11.994 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi028.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@smithi049.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-17T23:46:11.995 DEBUG:teuthology.orchestra.run.smithi049:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2024-09-17T23:46:12.047 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-17T23:46:12.047 INFO:tasks.ceph:role: cluster2.osd.0 2024-09-17T23:46:12.048 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi049.front.sepia.ceph.com 2024-09-17T23:46:12.048 DEBUG:teuthology.orchestra.run.smithi049:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-09-17T23:46:12.104 INFO:teuthology.orchestra.run.smithi049.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-09-17T23:46:12.104 INFO:teuthology.orchestra.run.smithi049.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-17T23:46:12.104 INFO:teuthology.orchestra.run.smithi049.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-17T23:46:12.105 INFO:teuthology.orchestra.run.smithi049.stdout: = reflink=1 bigtime=0 inobtcount=0 2024-09-17T23:46:12.105 INFO:teuthology.orchestra.run.smithi049.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-17T23:46:12.105 INFO:teuthology.orchestra.run.smithi049.stdout: = sunit=0 swidth=0 blks 2024-09-17T23:46:12.105 INFO:teuthology.orchestra.run.smithi049.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-17T23:46:12.105 INFO:teuthology.orchestra.run.smithi049.stdout:log =internal log bsize=4096 blocks=11443, version=2 2024-09-17T23:46:12.105 INFO:teuthology.orchestra.run.smithi049.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-17T23:46:12.105 INFO:teuthology.orchestra.run.smithi049.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-17T23:46:12.128 INFO:teuthology.orchestra.run.smithi049.stdout:Discarding blocks...Done. 2024-09-17T23:46:12.130 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi049.front.sepia.ceph.com -o noatime 2024-09-17T23:46:12.130 DEBUG:teuthology.orchestra.run.smithi049:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2024-09-17T23:46:12.187 DEBUG:teuthology.orchestra.run.smithi049:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2024-09-17T23:46:12.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T23:46:12.242 INFO:teuthology.orchestra.run.smithi049.stderr:sudo: /sbin/restorecon: command not found 2024-09-17T23:46:12.243 DEBUG:teuthology.orchestra.run.smithi049:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2024-09-17T23:46:12.299 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-17T23:46:12.300 INFO:tasks.ceph:role: cluster2.osd.1 2024-09-17T23:46:12.300 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi049.front.sepia.ceph.com 2024-09-17T23:46:12.300 DEBUG:teuthology.orchestra.run.smithi049:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-09-17T23:46:12.360 INFO:teuthology.orchestra.run.smithi049.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-09-17T23:46:12.361 INFO:teuthology.orchestra.run.smithi049.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-17T23:46:12.361 INFO:teuthology.orchestra.run.smithi049.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-17T23:46:12.361 INFO:teuthology.orchestra.run.smithi049.stdout: = reflink=1 bigtime=0 inobtcount=0 2024-09-17T23:46:12.361 INFO:teuthology.orchestra.run.smithi049.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-17T23:46:12.361 INFO:teuthology.orchestra.run.smithi049.stdout: = sunit=0 swidth=0 blks 2024-09-17T23:46:12.361 INFO:teuthology.orchestra.run.smithi049.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-17T23:46:12.361 INFO:teuthology.orchestra.run.smithi049.stdout:log =internal log bsize=4096 blocks=11443, version=2 2024-09-17T23:46:12.361 INFO:teuthology.orchestra.run.smithi049.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-17T23:46:12.361 INFO:teuthology.orchestra.run.smithi049.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-17T23:46:12.384 INFO:teuthology.orchestra.run.smithi049.stdout:Discarding blocks...Done. 2024-09-17T23:46:12.386 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi049.front.sepia.ceph.com -o noatime 2024-09-17T23:46:12.387 DEBUG:teuthology.orchestra.run.smithi049:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2024-09-17T23:46:12.445 DEBUG:teuthology.orchestra.run.smithi049:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2024-09-17T23:46:12.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T23:46:12.499 INFO:teuthology.orchestra.run.smithi049.stderr:sudo: /sbin/restorecon: command not found 2024-09-17T23:46:12.499 DEBUG:teuthology.orchestra.run.smithi049:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2024-09-17T23:46:12.553 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-17T23:46:12.553 INFO:tasks.ceph:role: cluster2.osd.2 2024-09-17T23:46:12.553 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi049.front.sepia.ceph.com 2024-09-17T23:46:12.553 DEBUG:teuthology.orchestra.run.smithi049:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-09-17T23:46:12.608 INFO:teuthology.orchestra.run.smithi049.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-09-17T23:46:12.608 INFO:teuthology.orchestra.run.smithi049.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-17T23:46:12.608 INFO:teuthology.orchestra.run.smithi049.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-17T23:46:12.608 INFO:teuthology.orchestra.run.smithi049.stdout: = reflink=1 bigtime=0 inobtcount=0 2024-09-17T23:46:12.608 INFO:teuthology.orchestra.run.smithi049.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-17T23:46:12.608 INFO:teuthology.orchestra.run.smithi049.stdout: = sunit=0 swidth=0 blks 2024-09-17T23:46:12.608 INFO:teuthology.orchestra.run.smithi049.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-17T23:46:12.609 INFO:teuthology.orchestra.run.smithi049.stdout:log =internal log bsize=4096 blocks=11443, version=2 2024-09-17T23:46:12.609 INFO:teuthology.orchestra.run.smithi049.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-17T23:46:12.609 INFO:teuthology.orchestra.run.smithi049.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-17T23:46:12.631 INFO:teuthology.orchestra.run.smithi049.stdout:Discarding blocks...Done. 2024-09-17T23:46:12.633 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi049.front.sepia.ceph.com -o noatime 2024-09-17T23:46:12.633 DEBUG:teuthology.orchestra.run.smithi049:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2024-09-17T23:46:12.696 DEBUG:teuthology.orchestra.run.smithi049:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2024-09-17T23:46:12.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T23:46:12.751 INFO:teuthology.orchestra.run.smithi049.stderr:sudo: /sbin/restorecon: command not found 2024-09-17T23:46:12.752 DEBUG:teuthology.orchestra.run.smithi049:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-09-17T23:46:12.861 INFO:teuthology.orchestra.run.smithi049.stderr:2024-09-17T23:46:12.859+0000 7fbb550947c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2024-09-17T23:46:12.861 INFO:teuthology.orchestra.run.smithi049.stderr:2024-09-17T23:46:12.859+0000 7fbb550947c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2024-09-17T23:46:12.861 INFO:teuthology.orchestra.run.smithi049.stderr:2024-09-17T23:46:12.859+0000 7fbb550947c0 -1 bdev(0x56022370f800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2024-09-17T23:46:12.861 INFO:teuthology.orchestra.run.smithi049.stderr:2024-09-17T23:46:12.859+0000 7fbb550947c0 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2024-09-17T23:46:15.993 DEBUG:teuthology.orchestra.run.smithi049:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2024-09-17T23:46:16.007 DEBUG:teuthology.orchestra.run.smithi049:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-09-17T23:46:16.089 INFO:teuthology.orchestra.run.smithi049.stderr:2024-09-17T23:46:16.087+0000 7f97ebf6e7c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2024-09-17T23:46:16.089 INFO:teuthology.orchestra.run.smithi049.stderr:2024-09-17T23:46:16.087+0000 7f97ebf6e7c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2024-09-17T23:46:16.089 INFO:teuthology.orchestra.run.smithi049.stderr:2024-09-17T23:46:16.087+0000 7f97ebf6e7c0 -1 bdev(0x55fbe2cbb800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2024-09-17T23:46:16.089 INFO:teuthology.orchestra.run.smithi049.stderr:2024-09-17T23:46:16.087+0000 7f97ebf6e7c0 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2024-09-17T23:46:19.230 DEBUG:teuthology.orchestra.run.smithi049:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2024-09-17T23:46:19.286 DEBUG:teuthology.orchestra.run.smithi049:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-09-17T23:46:19.370 INFO:teuthology.orchestra.run.smithi049.stderr:2024-09-17T23:46:19.367+0000 7f3baf15e7c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2024-09-17T23:46:19.370 INFO:teuthology.orchestra.run.smithi049.stderr:2024-09-17T23:46:19.367+0000 7f3baf15e7c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2024-09-17T23:46:19.370 INFO:teuthology.orchestra.run.smithi049.stderr:2024-09-17T23:46:19.367+0000 7f3baf15e7c0 -1 bdev(0x55e5c57cd800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2024-09-17T23:46:19.370 INFO:teuthology.orchestra.run.smithi049.stderr:2024-09-17T23:46:19.367+0000 7f3baf15e7c0 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2024-09-17T23:46:22.519 DEBUG:teuthology.orchestra.run.smithi049:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2024-09-17T23:46:22.572 INFO:tasks.ceph:Reading keys from all nodes... 2024-09-17T23:46:22.573 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:46:22.573 DEBUG:teuthology.orchestra.run.smithi028:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2024-09-17T23:46:22.588 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:22.588 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2024-09-17T23:46:22.628 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:22.628 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2024-09-17T23:46:22.685 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:22.685 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2024-09-17T23:46:22.740 DEBUG:teuthology.orchestra.run.smithi028:> set -ex 2024-09-17T23:46:22.740 DEBUG:teuthology.orchestra.run.smithi028:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2024-09-17T23:46:22.746 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:22.746 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2024-09-17T23:46:22.788 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:22.789 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2024-09-17T23:46:22.837 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:22.837 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2024-09-17T23:46:22.885 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:22.885 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2024-09-17T23:46:22.933 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:22.933 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2024-09-17T23:46:22.981 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:22.981 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2024-09-17T23:46:23.029 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:23.029 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2024-09-17T23:46:23.076 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:23.077 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2024-09-17T23:46:23.125 INFO:tasks.ceph:Adding keys to all mons... 2024-09-17T23:46:23.125 DEBUG:teuthology.orchestra.run.smithi049:> sudo tee -a /etc/ceph/cluster2.keyring 2024-09-17T23:46:23.179 INFO:teuthology.orchestra.run.smithi049.stdout:[mgr.x] 2024-09-17T23:46:23.179 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpm/tpCDhAAZJL+ZGUBh1UK7KQa4Moagw== 2024-09-17T23:46:23.179 INFO:teuthology.orchestra.run.smithi049.stdout:[osd.0] 2024-09-17T23:46:23.179 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDEFOpmIPdYMxAAPXQIKpLd2IaHsizXMUgS5w== 2024-09-17T23:46:23.179 INFO:teuthology.orchestra.run.smithi049.stdout:[osd.1] 2024-09-17T23:46:23.179 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDIFOpmLjJYBRAAhr84/BX69Cc4dHXMi0EnbA== 2024-09-17T23:46:23.179 INFO:teuthology.orchestra.run.smithi049.stdout:[osd.2] 2024-09-17T23:46:23.179 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDLFOpm0iARFhAAfn1SoEe7xlXQ/3BDN0J6yA== 2024-09-17T23:46:23.179 INFO:teuthology.orchestra.run.smithi049.stdout:[client.0] 2024-09-17T23:46:23.179 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmYYwLExAAjYNVn1HHJJSh4RnXZVsOFA== 2024-09-17T23:46:23.179 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror] 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmScDjFRAATNwzmXAGI5s/bBlVKDFRNQ== 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.0] 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmNYsnGxAAdzoj2zRP6aGtSPquqYl9sg== 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.1] 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmxAJSIBAA3PJf/c5BfKF3ueQontpiLg== 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.2] 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmY96LJRAAk/lDVjGCKbBKIUYIZxXjhw== 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.3] 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmcQkKKxAAJhym19FFX+safELCaQD4PA== 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.4] 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmBJFsMBAAwU0KFwlVG32dnhBoWVpuKw== 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.5] 2024-09-17T23:46:23.180 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmH719NRAAxx3ntpMdnxSyRXZaxAxm1Q== 2024-09-17T23:46:23.181 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.6] 2024-09-17T23:46:23.181 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmkp65OhAA6CT6KX+G8GU3qxDJdfk6uA== 2024-09-17T23:46:23.181 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-17T23:46:23.271 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-17T23:46:23.316 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-17T23:46:23.405 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-17T23:46:23.499 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:46:23.549 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:46:23.593 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:46:23.678 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:46:23.764 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:46:23.856 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:46:23.903 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:46:23.945 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:46:24.030 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-17T23:46:24.116 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-09-17T23:46:24.117 DEBUG:teuthology.orchestra.run.smithi049:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2024-09-17T23:46:24.127 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2024-09-17T23:46:24.219 DEBUG:teuthology.orchestra.run.smithi049:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2024-09-17T23:46:24.230 DEBUG:teuthology.orchestra.run.smithi049:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2024-09-17T23:46:24.275 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2024-09-17T23:46:24.276 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2024-09-17T23:46:24.276 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2024-09-17T23:46:24.321 INFO:tasks.ceph.cluster2.mon.a:Started 2024-09-17T23:46:24.321 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2024-09-17T23:46:24.322 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2024-09-17T23:46:24.322 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2024-09-17T23:46:24.324 INFO:tasks.ceph.cluster2.mgr.x:Started 2024-09-17T23:46:24.325 INFO:tasks.ceph:Setting crush tunables to default 2024-09-17T23:46:24.325 DEBUG:teuthology.orchestra.run.smithi049:> sudo ceph --cluster cluster2 osd crush tunables default 2024-09-17T23:46:24.551 INFO:teuthology.orchestra.run.smithi049.stderr:adjusted tunables profile to default 2024-09-17T23:46:24.568 INFO:tasks.ceph:check_enable_crimson: False 2024-09-17T23:46:24.569 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2024-09-17T23:46:24.569 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:24.569 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2024-09-17T23:46:24.580 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:24.580 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2024-09-17T23:46:24.634 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:46:24.634 DEBUG:teuthology.orchestra.run.smithi049:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2024-09-17T23:46:24.685 DEBUG:teuthology.orchestra.run.smithi049:> sudo ceph --cluster cluster2 osd new 6fde4361-75a8-478d-b104-6083fe26abe3 0 2024-09-17T23:46:24.721 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:24.718+0000 7f62ae961180 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T23:46:24.936 INFO:teuthology.orchestra.run.smithi049.stdout:0 2024-09-17T23:46:24.954 DEBUG:teuthology.orchestra.run.smithi049:> sudo ceph --cluster cluster2 osd new b54753b9-dd30-4601-93f0-bf45a643a1d2 1 2024-09-17T23:46:25.125 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:25.122+0000 7f62ae961180 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T23:46:25.163 INFO:teuthology.orchestra.run.smithi049.stdout:1 2024-09-17T23:46:25.182 DEBUG:teuthology.orchestra.run.smithi049:> sudo ceph --cluster cluster2 osd new 5fdd2e96-616c-48e2-8473-0f00c1016b6c 2 2024-09-17T23:46:25.268 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:25.266+0000 7f62ae961180 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T23:46:25.380 INFO:teuthology.orchestra.run.smithi049.stdout:2 2024-09-17T23:46:25.398 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2024-09-17T23:46:25.398 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2024-09-17T23:46:25.400 INFO:tasks.ceph.cluster2.osd.0:Started 2024-09-17T23:46:25.400 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2024-09-17T23:46:25.400 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2024-09-17T23:46:25.403 INFO:tasks.ceph.cluster2.osd.1:Started 2024-09-17T23:46:25.403 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2024-09-17T23:46:25.403 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2024-09-17T23:46:25.406 INFO:tasks.ceph.cluster2.osd.2:Started 2024-09-17T23:46:25.407 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-17T23:46:25.592 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:25.592 INFO:teuthology.orchestra.run.smithi049.stdout:{"epoch":5,"fsid":"024a7a4f-3e37-4953-ac7d-ad33b26dedaa","created":"2024-09-17T23:46:24.453844+0000","modified":"2024-09-17T23:46:25.380066+0000","last_up_change":"0.000000","last_in_change":"2024-09-17T23:46:25.380066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"6fde4361-75a8-478d-b104-6083fe26abe3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"b54753b9-dd30-4601-93f0-bf45a643a1d2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"5fdd2e96-616c-48e2-8473-0f00c1016b6c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T23:46:25.610 INFO:tasks.ceph.ceph_manager.cluster2:[] 2024-09-17T23:46:25.610 INFO:tasks.ceph:Waiting for OSDs to come up 2024-09-17T23:46:25.705 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:25.702+0000 7f62ae961180 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T23:46:25.758 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:25.754+0000 7f62ae961180 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T23:46:25.809 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:25.806+0000 7f62ae961180 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T23:46:25.885 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:25.882+0000 7f62ae961180 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T23:46:25.912 DEBUG:teuthology.orchestra.run.smithi049:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-09-17T23:46:25.936 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:25.934+0000 7f62ae961180 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T23:46:26.012 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:26.010+0000 7f62ae961180 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T23:46:26.064 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:26.062+0000 7f62ae961180 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T23:46:26.105 INFO:teuthology.misc.health.smithi049.stdout: 2024-09-17T23:46:26.105 INFO:teuthology.misc.health.smithi049.stdout:{"epoch":5,"fsid":"024a7a4f-3e37-4953-ac7d-ad33b26dedaa","created":"2024-09-17T23:46:24.453844+0000","modified":"2024-09-17T23:46:25.380066+0000","last_up_change":"0.000000","last_in_change":"2024-09-17T23:46:25.380066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"6fde4361-75a8-478d-b104-6083fe26abe3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"b54753b9-dd30-4601-93f0-bf45a643a1d2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"5fdd2e96-616c-48e2-8473-0f00c1016b6c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T23:46:26.122 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-17T23:46:26.208 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:26.206+0000 7f62ae961180 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T23:46:26.261 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:26.258+0000 7f62ae961180 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T23:46:26.486 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2024-09-17T23:46:26.486 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2024-09-17T23:46:26.486 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr: from numpy import show_config as show_numpy_config 2024-09-17T23:46:26.493 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:26.490+0000 7f62ae961180 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T23:46:26.534 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-17T23:46:26.531+0000 7f52e827d7c0 -1 Falling back to public interface 2024-09-17T23:46:26.633 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:26.630+0000 7f62ae961180 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T23:46:26.746 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:26.742+0000 7f62ae961180 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T23:46:26.794 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-17T23:46:26.791+0000 7fd4d1b9c7c0 -1 Falling back to public interface 2024-09-17T23:46:26.809 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:26.806+0000 7f62ae961180 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T23:46:26.949 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:26.946+0000 7f62ae961180 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T23:46:27.018 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:46:27.015+0000 7fbbf912a7c0 -1 Falling back to public interface 2024-09-17T23:46:27.443 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:27.442+0000 7f62ae961180 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T23:46:27.519 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:27.518+0000 7f62ae961180 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T23:46:27.669 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:27.666+0000 7f62ae961180 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T23:46:27.745 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:27.742+0000 7f62ae961180 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T23:46:28.040 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:28.038+0000 7f62ae961180 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T23:46:28.096 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:28.094+0000 7f62ae961180 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T23:46:29.112 INFO:tasks.ceph.cluster2.mgr.x.smithi028.stderr:2024-09-17T23:46:29.110+0000 7f6255598640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-17T23:46:30.520 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-17T23:46:30.519+0000 7f52e827d7c0 -1 osd.0 0 log_to_monitors true 2024-09-17T23:46:30.980 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-17T23:46:30.979+0000 7fd4d1b9c7c0 -1 osd.2 0 log_to_monitors true 2024-09-17T23:46:31.206 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:46:31.203+0000 7fbbf912a7c0 -1 osd.1 0 log_to_monitors true 2024-09-17T23:46:32.426 DEBUG:teuthology.orchestra.run.smithi049:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-09-17T23:46:32.874 INFO:teuthology.misc.health.smithi049.stdout: 2024-09-17T23:46:32.874 INFO:teuthology.misc.health.smithi049.stdout:{"epoch":7,"fsid":"024a7a4f-3e37-4953-ac7d-ad33b26dedaa","created":"2024-09-17T23:46:24.453844+0000","modified":"2024-09-17T23:46:32.165631+0000","last_up_change":"0.000000","last_in_change":"2024-09-17T23:46:25.380066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"6fde4361-75a8-478d-b104-6083fe26abe3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"b54753b9-dd30-4601-93f0-bf45a643a1d2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"5fdd2e96-616c-48e2-8473-0f00c1016b6c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T23:46:32.897 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-17T23:46:33.315 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-17T23:46:33.311+0000 7fd4cdb45640 -1 osd.2 0 waiting for initial osdmap 2024-09-17T23:46:33.321 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-17T23:46:33.319+0000 7f52e4226640 -1 osd.0 0 waiting for initial osdmap 2024-09-17T23:46:33.345 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-17T23:46:33.343+0000 7fd4c597a640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T23:46:33.361 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-17T23:46:33.359+0000 7f52dc05b640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T23:46:34.050 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:46:34.047+0000 7fbbf50d3640 -1 osd.1 0 waiting for initial osdmap 2024-09-17T23:46:34.072 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:46:34.071+0000 7fbbecf08640 -1 osd.1 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T23:46:39.201 DEBUG:teuthology.orchestra.run.smithi049:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-09-17T23:46:39.506 INFO:teuthology.misc.health.smithi049.stdout: 2024-09-17T23:46:39.507 INFO:teuthology.misc.health.smithi049.stdout:{"epoch":13,"fsid":"024a7a4f-3e37-4953-ac7d-ad33b26dedaa","created":"2024-09-17T23:46:24.453844+0000","modified":"2024-09-17T23:46:38.338299+0000","last_up_change":"2024-09-17T23:46:34.324287+0000","last_in_change":"2024-09-17T23:46:25.380066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T23:46:36.186072+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6fde4361-75a8-478d-b104-6083fe26abe3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6800","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6801","nonce":687973982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6802","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6803","nonce":687973982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6806","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6807","nonce":687973982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6804","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6805","nonce":687973982}]},"public_addr":"172.21.15.49:6801/687973982","cluster_addr":"172.21.15.49:6803/687973982","heartbeat_back_addr":"172.21.15.49:6807/687973982","heartbeat_front_addr":"172.21.15.49:6805/687973982","state":["exists","up"]},{"osd":1,"uuid":"b54753b9-dd30-4601-93f0-bf45a643a1d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6816","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6817","nonce":965597954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6818","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6819","nonce":965597954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6822","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6823","nonce":965597954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6820","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6821","nonce":965597954}]},"public_addr":"172.21.15.49:6817/965597954","cluster_addr":"172.21.15.49:6819/965597954","heartbeat_back_addr":"172.21.15.49:6823/965597954","heartbeat_front_addr":"172.21.15.49:6821/965597954","state":["exists","up"]},{"osd":2,"uuid":"5fdd2e96-616c-48e2-8473-0f00c1016b6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6808","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6809","nonce":130731046}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6810","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6811","nonce":130731046}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6814","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6815","nonce":130731046}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6812","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6813","nonce":130731046}]},"public_addr":"172.21.15.49:6809/130731046","cluster_addr":"172.21.15.49:6811/130731046","heartbeat_back_addr":"172.21.15.49:6815/130731046","heartbeat_front_addr":"172.21.15.49:6813/130731046","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:46:31.554586+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:46:32.203336+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:46:31.945325+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T23:46:39.524 DEBUG:teuthology.misc:3 of 3 OSDs are up 2024-09-17T23:46:39.524 INFO:tasks.ceph:Creating RBD pool 2024-09-17T23:46:39.525 DEBUG:teuthology.orchestra.run.smithi049:> sudo ceph --cluster cluster2 osd pool create rbd 8 2024-09-17T23:46:40.347 INFO:teuthology.orchestra.run.smithi049.stderr:pool 'rbd' created 2024-09-17T23:46:40.376 DEBUG:teuthology.orchestra.run.smithi049:> rbd --cluster cluster2 pool init rbd 2024-09-17T23:46:43.382 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2024-09-17T23:46:43.383 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2024-09-17T23:46:43.383 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2024-09-17T23:46:43.384 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2024-09-17T23:46:43.384 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-09-17T23:46:43.746 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:43.765 INFO:teuthology.orchestra.run.smithi049.stdout:{"epoch":5,"flags":0,"active_gid":4101,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6826","nonce":2572959553},{"type":"v1","addr":"172.21.15.28:6827","nonce":2572959553}]},"active_addr":"172.21.15.28:6827/2572959553","active_change":"2024-09-17T23:46:28.099110+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":2151842443}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":1656603269}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":3829534519}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.28:0","nonce":1786733027}]}]} 2024-09-17T23:46:43.769 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2024-09-17T23:46:43.769 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2024-09-17T23:46:43.769 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-17T23:46:44.069 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:44.069 INFO:teuthology.orchestra.run.smithi049.stdout:{"epoch":17,"fsid":"024a7a4f-3e37-4953-ac7d-ad33b26dedaa","created":"2024-09-17T23:46:24.453844+0000","modified":"2024-09-17T23:46:43.358903+0000","last_up_change":"2024-09-17T23:46:34.324287+0000","last_in_change":"2024-09-17T23:46:25.380066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T23:46:36.186072+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-17T23:46:39.870458+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6fde4361-75a8-478d-b104-6083fe26abe3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6800","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6801","nonce":687973982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6802","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6803","nonce":687973982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6806","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6807","nonce":687973982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6804","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6805","nonce":687973982}]},"public_addr":"172.21.15.49:6801/687973982","cluster_addr":"172.21.15.49:6803/687973982","heartbeat_back_addr":"172.21.15.49:6807/687973982","heartbeat_front_addr":"172.21.15.49:6805/687973982","state":["exists","up"]},{"osd":1,"uuid":"b54753b9-dd30-4601-93f0-bf45a643a1d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6816","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6817","nonce":965597954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6818","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6819","nonce":965597954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6822","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6823","nonce":965597954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6820","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6821","nonce":965597954}]},"public_addr":"172.21.15.49:6817/965597954","cluster_addr":"172.21.15.49:6819/965597954","heartbeat_back_addr":"172.21.15.49:6823/965597954","heartbeat_front_addr":"172.21.15.49:6821/965597954","state":["exists","up"]},{"osd":2,"uuid":"5fdd2e96-616c-48e2-8473-0f00c1016b6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6808","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6809","nonce":130731046}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6810","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6811","nonce":130731046}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6814","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6815","nonce":130731046}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6812","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6813","nonce":130731046}]},"public_addr":"172.21.15.49:6809/130731046","cluster_addr":"172.21.15.49:6811/130731046","heartbeat_back_addr":"172.21.15.49:6815/130731046","heartbeat_front_addr":"172.21.15.49:6813/130731046","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:46:31.554586+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:46:32.203336+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:46:31.945325+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T23:46:44.086 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2024-09-17T23:46:44.086 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-17T23:46:44.373 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:44.373 INFO:teuthology.orchestra.run.smithi049.stdout:{"epoch":17,"fsid":"024a7a4f-3e37-4953-ac7d-ad33b26dedaa","created":"2024-09-17T23:46:24.453844+0000","modified":"2024-09-17T23:46:43.358903+0000","last_up_change":"2024-09-17T23:46:34.324287+0000","last_in_change":"2024-09-17T23:46:25.380066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T23:46:36.186072+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-17T23:46:39.870458+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6fde4361-75a8-478d-b104-6083fe26abe3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6800","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6801","nonce":687973982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6802","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6803","nonce":687973982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6806","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6807","nonce":687973982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6804","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6805","nonce":687973982}]},"public_addr":"172.21.15.49:6801/687973982","cluster_addr":"172.21.15.49:6803/687973982","heartbeat_back_addr":"172.21.15.49:6807/687973982","heartbeat_front_addr":"172.21.15.49:6805/687973982","state":["exists","up"]},{"osd":1,"uuid":"b54753b9-dd30-4601-93f0-bf45a643a1d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6816","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6817","nonce":965597954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6818","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6819","nonce":965597954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6822","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6823","nonce":965597954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6820","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6821","nonce":965597954}]},"public_addr":"172.21.15.49:6817/965597954","cluster_addr":"172.21.15.49:6819/965597954","heartbeat_back_addr":"172.21.15.49:6823/965597954","heartbeat_front_addr":"172.21.15.49:6821/965597954","state":["exists","up"]},{"osd":2,"uuid":"5fdd2e96-616c-48e2-8473-0f00c1016b6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6808","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6809","nonce":130731046}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6810","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6811","nonce":130731046}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6814","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6815","nonce":130731046}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6812","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6813","nonce":130731046}]},"public_addr":"172.21.15.49:6809/130731046","cluster_addr":"172.21.15.49:6811/130731046","heartbeat_back_addr":"172.21.15.49:6815/130731046","heartbeat_front_addr":"172.21.15.49:6813/130731046","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:46:31.554586+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:46:32.203336+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:46:31.945325+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T23:46:44.391 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2024-09-17T23:46:44.391 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2024-09-17T23:46:44.392 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2024-09-17T23:46:44.545 INFO:teuthology.orchestra.run.smithi049.stdout:38654705670 2024-09-17T23:46:44.545 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2024-09-17T23:46:44.558 INFO:teuthology.orchestra.run.smithi049.stdout:38654705670 2024-09-17T23:46:44.559 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2024-09-17T23:46:44.560 INFO:teuthology.orchestra.run.smithi049.stdout:38654705668 2024-09-17T23:46:44.561 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2024-09-17T23:46:44.830 INFO:teuthology.orchestra.run.smithi049.stdout:38654705669 2024-09-17T23:46:44.847 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705670 got 38654705669 for osd.0 2024-09-17T23:46:44.883 INFO:teuthology.orchestra.run.smithi049.stdout:38654705669 2024-09-17T23:46:44.895 INFO:teuthology.orchestra.run.smithi049.stdout:38654705667 2024-09-17T23:46:44.901 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705670 got 38654705669 for osd.2 2024-09-17T23:46:44.912 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705667 for osd.1 2024-09-17T23:46:45.848 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2024-09-17T23:46:45.902 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2024-09-17T23:46:45.913 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2024-09-17T23:46:46.161 INFO:teuthology.orchestra.run.smithi049.stdout:38654705669 2024-09-17T23:46:46.178 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705670 got 38654705669 for osd.0 2024-09-17T23:46:46.222 INFO:teuthology.orchestra.run.smithi049.stdout:38654705669 2024-09-17T23:46:46.239 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705670 got 38654705669 for osd.2 2024-09-17T23:46:46.259 INFO:teuthology.orchestra.run.smithi049.stdout:38654705667 2024-09-17T23:46:46.276 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705667 for osd.1 2024-09-17T23:46:47.179 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2024-09-17T23:46:47.240 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2024-09-17T23:46:47.276 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2024-09-17T23:46:47.494 INFO:teuthology.orchestra.run.smithi049.stdout:38654705671 2024-09-17T23:46:47.512 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705670 got 38654705671 for osd.0 2024-09-17T23:46:47.512 DEBUG:teuthology.parallel:result is None 2024-09-17T23:46:47.536 INFO:teuthology.orchestra.run.smithi049.stdout:38654705671 2024-09-17T23:46:47.553 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705670 got 38654705671 for osd.2 2024-09-17T23:46:47.553 DEBUG:teuthology.parallel:result is None 2024-09-17T23:46:47.590 INFO:teuthology.orchestra.run.smithi049.stdout:38654705669 2024-09-17T23:46:47.607 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705669 for osd.1 2024-09-17T23:46:47.608 DEBUG:teuthology.parallel:result is None 2024-09-17T23:46:47.608 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2024-09-17T23:46:47.608 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-17T23:46:47.935 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:47.936 INFO:teuthology.orchestra.run.smithi049.stderr:dumped all 2024-09-17T23:46:47.953 INFO:teuthology.orchestra.run.smithi049.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2024-09-17T23:46:46.110493+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":103496,"kb_used_data":1600,"kb_used_omap":4,"kb_used_meta":101819,"kb_avail":283012024,"statfs":{"total":289910292480,"available":289804312576,"internally_reserved":0,"allocated":1638400,"data_stored":1309576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":104262991},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.761208"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.368792+0000","last_change":"2024-09-17T23:46:43.368792+0000","last_active":"2024-09-17T23:46:43.368792+0000","last_peered":"2024-09-17T23:46:43.368792+0000","last_clean":"2024-09-17T23:46:43.368792+0000","last_became_active":"2024-09-17T23:46:41.368790+0000","last_became_peered":"2024-09-17T23:46:41.368790+0000","last_unstale":"2024-09-17T23:46:43.368792+0000","last_undegraded":"2024-09-17T23:46:43.368792+0000","last_fullsized":"2024-09-17T23:46:43.368792+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:34:56.033504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012361539999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.369747+0000","last_change":"2024-09-17T23:46:43.369747+0000","last_active":"2024-09-17T23:46:43.369747+0000","last_peered":"2024-09-17T23:46:43.369747+0000","last_clean":"2024-09-17T23:46:43.369747+0000","last_became_active":"2024-09-17T23:46:41.367907+0000","last_became_peered":"2024-09-17T23:46:41.367907+0000","last_unstale":"2024-09-17T23:46:43.369747+0000","last_undegraded":"2024-09-17T23:46:43.369747+0000","last_fullsized":"2024-09-17T23:46:43.369747+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:35:19.220254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002060517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.369633+0000","last_change":"2024-09-17T23:46:43.369633+0000","last_active":"2024-09-17T23:46:43.369633+0000","last_peered":"2024-09-17T23:46:43.369633+0000","last_clean":"2024-09-17T23:46:43.369633+0000","last_became_active":"2024-09-17T23:46:41.367679+0000","last_became_peered":"2024-09-17T23:46:41.367679+0000","last_unstale":"2024-09-17T23:46:43.369633+0000","last_undegraded":"2024-09-17T23:46:43.369633+0000","last_fullsized":"2024-09-17T23:46:43.369633+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:46:40.205238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020925570000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.368634+0000","last_change":"2024-09-17T23:46:43.368634+0000","last_active":"2024-09-17T23:46:43.368634+0000","last_peered":"2024-09-17T23:46:43.368634+0000","last_clean":"2024-09-17T23:46:43.368634+0000","last_became_active":"2024-09-17T23:46:41.368202+0000","last_became_peered":"2024-09-17T23:46:41.368202+0000","last_unstale":"2024-09-17T23:46:43.368634+0000","last_undegraded":"2024-09-17T23:46:43.368634+0000","last_fullsized":"2024-09-17T23:46:43.368634+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:30:19.644528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011726320000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.374596+0000","last_change":"2024-09-17T23:46:43.373705+0000","last_active":"2024-09-17T23:46:43.374596+0000","last_peered":"2024-09-17T23:46:43.374596+0000","last_clean":"2024-09-17T23:46:43.374596+0000","last_became_active":"2024-09-17T23:46:41.366093+0000","last_became_peered":"2024-09-17T23:46:41.366093+0000","last_unstale":"2024-09-17T23:46:43.374596+0000","last_undegraded":"2024-09-17T23:46:43.374596+0000","last_fullsized":"2024-09-17T23:46:43.374596+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T00:00:49.511275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0021178640000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.865198+0000","last_change":"2024-09-17T23:46:43.865198+0000","last_active":"2024-09-17T23:46:43.865198+0000","last_peered":"2024-09-17T23:46:43.865198+0000","last_clean":"2024-09-17T23:46:43.865198+0000","last_became_active":"2024-09-17T23:46:41.367488+0000","last_became_peered":"2024-09-17T23:46:41.367488+0000","last_unstale":"2024-09-17T23:46:43.865198+0000","last_undegraded":"2024-09-17T23:46:43.865198+0000","last_fullsized":"2024-09-17T23:46:43.865198+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:39:09.298304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011158380000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.865240+0000","last_change":"2024-09-17T23:46:43.865240+0000","last_active":"2024-09-17T23:46:43.865240+0000","last_peered":"2024-09-17T23:46:43.865240+0000","last_clean":"2024-09-17T23:46:43.865240+0000","last_became_active":"2024-09-17T23:46:41.365093+0000","last_became_peered":"2024-09-17T23:46:41.365093+0000","last_unstale":"2024-09-17T23:46:43.865240+0000","last_undegraded":"2024-09-17T23:46:43.865240+0000","last_fullsized":"2024-09-17T23:46:43.865240+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T23:46:54.488828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011454340000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.368031+0000","last_change":"2024-09-17T23:46:43.368031+0000","last_active":"2024-09-17T23:46:43.368031+0000","last_peered":"2024-09-17T23:46:43.368031+0000","last_clean":"2024-09-17T23:46:43.368031+0000","last_became_active":"2024-09-17T23:46:41.364533+0000","last_became_peered":"2024-09-17T23:46:41.364533+0000","last_unstale":"2024-09-17T23:46:43.368031+0000","last_undegraded":"2024-09-17T23:46:43.368031+0000","last_fullsized":"2024-09-17T23:46:43.368031+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:27:19.243264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077234799999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'97","reported_seq":147,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.367147+0000","last_change":"2024-09-17T23:46:37.589635+0000","last_active":"2024-09-17T23:46:43.367147+0000","last_peered":"2024-09-17T23:46:43.367147+0000","last_clean":"2024-09-17T23:46:43.367147+0000","last_became_active":"2024-09-17T23:46:37.587086+0000","last_became_peered":"2024-09-17T23:46:37.587086+0000","last_unstale":"2024-09-17T23:46:43.367147+0000","last_undegraded":"2024-09-17T23:46:43.367147+0000","last_fullsized":"2024-09-17T23:46:43.367147+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:36.327066+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:36.327066+0000","last_clean_scrub_stamp":"2024-09-17T23:46:36.327066+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:04:13.926025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705671,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":35496,"kb_used_data":144,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94336344,"statfs":{"total":96636764160,"available":96600416256,"internally_reserved":0,"allocated":147456,"data_stored":42934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705669,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31920,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339920,"statfs":{"total":96636764160,"available":96604078080,"internally_reserved":0,"allocated":745472,"data_stored":633321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705671,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36080,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335760,"statfs":{"total":96636764160,"available":96599818240,"internally_reserved":0,"allocated":745472,"data_stored":633321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T23:46:47.954 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-17T23:46:48.247 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:48.247 INFO:teuthology.orchestra.run.smithi049.stderr:dumped all 2024-09-17T23:46:48.264 INFO:teuthology.orchestra.run.smithi049.stdout:{"pg_ready":true,"pg_map":{"version":22,"stamp":"2024-09-17T23:46:48.111000+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":103496,"kb_used_data":1600,"kb_used_omap":4,"kb_used_meta":101819,"kb_avail":283012024,"statfs":{"total":289910292480,"available":289804312576,"internally_reserved":0,"allocated":1638400,"data_stored":1309576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":104262991},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.756419"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.368792+0000","last_change":"2024-09-17T23:46:43.368792+0000","last_active":"2024-09-17T23:46:43.368792+0000","last_peered":"2024-09-17T23:46:43.368792+0000","last_clean":"2024-09-17T23:46:43.368792+0000","last_became_active":"2024-09-17T23:46:41.368790+0000","last_became_peered":"2024-09-17T23:46:41.368790+0000","last_unstale":"2024-09-17T23:46:43.368792+0000","last_undegraded":"2024-09-17T23:46:43.368792+0000","last_fullsized":"2024-09-17T23:46:43.368792+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:34:56.033504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012361539999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.369747+0000","last_change":"2024-09-17T23:46:43.369747+0000","last_active":"2024-09-17T23:46:43.369747+0000","last_peered":"2024-09-17T23:46:43.369747+0000","last_clean":"2024-09-17T23:46:43.369747+0000","last_became_active":"2024-09-17T23:46:41.367907+0000","last_became_peered":"2024-09-17T23:46:41.367907+0000","last_unstale":"2024-09-17T23:46:43.369747+0000","last_undegraded":"2024-09-17T23:46:43.369747+0000","last_fullsized":"2024-09-17T23:46:43.369747+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:35:19.220254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002060517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.369633+0000","last_change":"2024-09-17T23:46:43.369633+0000","last_active":"2024-09-17T23:46:43.369633+0000","last_peered":"2024-09-17T23:46:43.369633+0000","last_clean":"2024-09-17T23:46:43.369633+0000","last_became_active":"2024-09-17T23:46:41.367679+0000","last_became_peered":"2024-09-17T23:46:41.367679+0000","last_unstale":"2024-09-17T23:46:43.369633+0000","last_undegraded":"2024-09-17T23:46:43.369633+0000","last_fullsized":"2024-09-17T23:46:43.369633+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:46:40.205238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020925570000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.368634+0000","last_change":"2024-09-17T23:46:43.368634+0000","last_active":"2024-09-17T23:46:43.368634+0000","last_peered":"2024-09-17T23:46:43.368634+0000","last_clean":"2024-09-17T23:46:43.368634+0000","last_became_active":"2024-09-17T23:46:41.368202+0000","last_became_peered":"2024-09-17T23:46:41.368202+0000","last_unstale":"2024-09-17T23:46:43.368634+0000","last_undegraded":"2024-09-17T23:46:43.368634+0000","last_fullsized":"2024-09-17T23:46:43.368634+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:30:19.644528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011726320000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.374596+0000","last_change":"2024-09-17T23:46:43.373705+0000","last_active":"2024-09-17T23:46:43.374596+0000","last_peered":"2024-09-17T23:46:43.374596+0000","last_clean":"2024-09-17T23:46:43.374596+0000","last_became_active":"2024-09-17T23:46:41.366093+0000","last_became_peered":"2024-09-17T23:46:41.366093+0000","last_unstale":"2024-09-17T23:46:43.374596+0000","last_undegraded":"2024-09-17T23:46:43.374596+0000","last_fullsized":"2024-09-17T23:46:43.374596+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T00:00:49.511275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0021178640000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.865198+0000","last_change":"2024-09-17T23:46:43.865198+0000","last_active":"2024-09-17T23:46:43.865198+0000","last_peered":"2024-09-17T23:46:43.865198+0000","last_clean":"2024-09-17T23:46:43.865198+0000","last_became_active":"2024-09-17T23:46:41.367488+0000","last_became_peered":"2024-09-17T23:46:41.367488+0000","last_unstale":"2024-09-17T23:46:43.865198+0000","last_undegraded":"2024-09-17T23:46:43.865198+0000","last_fullsized":"2024-09-17T23:46:43.865198+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:39:09.298304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011158380000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.865240+0000","last_change":"2024-09-17T23:46:43.865240+0000","last_active":"2024-09-17T23:46:43.865240+0000","last_peered":"2024-09-17T23:46:43.865240+0000","last_clean":"2024-09-17T23:46:43.865240+0000","last_became_active":"2024-09-17T23:46:41.365093+0000","last_became_peered":"2024-09-17T23:46:41.365093+0000","last_unstale":"2024-09-17T23:46:43.865240+0000","last_undegraded":"2024-09-17T23:46:43.865240+0000","last_fullsized":"2024-09-17T23:46:43.865240+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T23:46:54.488828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011454340000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.368031+0000","last_change":"2024-09-17T23:46:43.368031+0000","last_active":"2024-09-17T23:46:43.368031+0000","last_peered":"2024-09-17T23:46:43.368031+0000","last_clean":"2024-09-17T23:46:43.368031+0000","last_became_active":"2024-09-17T23:46:41.364533+0000","last_became_peered":"2024-09-17T23:46:41.364533+0000","last_unstale":"2024-09-17T23:46:43.368031+0000","last_undegraded":"2024-09-17T23:46:43.368031+0000","last_fullsized":"2024-09-17T23:46:43.368031+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:27:19.243264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077234799999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'97","reported_seq":147,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-17T23:46:43.367147+0000","last_change":"2024-09-17T23:46:37.589635+0000","last_active":"2024-09-17T23:46:43.367147+0000","last_peered":"2024-09-17T23:46:43.367147+0000","last_clean":"2024-09-17T23:46:43.367147+0000","last_became_active":"2024-09-17T23:46:37.587086+0000","last_became_peered":"2024-09-17T23:46:37.587086+0000","last_unstale":"2024-09-17T23:46:43.367147+0000","last_undegraded":"2024-09-17T23:46:43.367147+0000","last_fullsized":"2024-09-17T23:46:43.367147+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:36.327066+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:36.327066+0000","last_clean_scrub_stamp":"2024-09-17T23:46:36.327066+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:04:13.926025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705671,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":35496,"kb_used_data":144,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94336344,"statfs":{"total":96636764160,"available":96600416256,"internally_reserved":0,"allocated":147456,"data_stored":42934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705669,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31920,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339920,"statfs":{"total":96636764160,"available":96604078080,"internally_reserved":0,"allocated":745472,"data_stored":633321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705671,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36080,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":35326,"kb_avail":94335760,"statfs":{"total":96636764160,"available":96599818240,"internally_reserved":0,"allocated":745472,"data_stored":633321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":36174277},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T23:46:48.265 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2024-09-17T23:46:48.265 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2024-09-17T23:46:48.265 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2024-09-17T23:46:48.265 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2024-09-17T23:46:48.587 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-17T23:46:48.587 INFO:teuthology.orchestra.run.smithi049.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-17T23:46:48.604 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2024-09-17T23:46:48.605 INFO:teuthology.run_tasks:Running task exec... 2024-09-17T23:46:48.618 INFO:teuthology.task.exec:Executing custom commands... 2024-09-17T23:46:48.618 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@smithi049.front.sepia.ceph.com 2024-09-17T23:46:48.618 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:48.997 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.4] 2024-09-17T23:46:48.997 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQCeFOpm51rzCRAAnK+BWXuAqGmv88f0inSWVQ== 2024-09-17T23:46:48.997 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:48.997 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:48.998 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.4 2024-09-17T23:46:49.018 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:49.389 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.5] 2024-09-17T23:46:49.390 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQCeFOpmrp0RDxAA1oLKpYO+MzrMg6eFwdxmRg== 2024-09-17T23:46:49.390 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:49.390 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:49.390 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.5 2024-09-17T23:46:49.408 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:49.788 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.6] 2024-09-17T23:46:49.788 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQCeFOpmYosVFBAADkGFhBcbaGjVa7XeKm2+hw== 2024-09-17T23:46:49.788 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:49.788 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:49.788 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.6 2024-09-17T23:46:49.808 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:50.165 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.4] 2024-09-17T23:46:50.165 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmBJFsMBAAwU0KFwlVG32dnhBoWVpuKw== 2024-09-17T23:46:50.165 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:50.165 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:50.166 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.4 2024-09-17T23:46:50.183 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:50.561 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.5] 2024-09-17T23:46:50.561 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmH719NRAAxx3ntpMdnxSyRXZaxAxm1Q== 2024-09-17T23:46:50.561 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:50.561 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:50.562 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.5 2024-09-17T23:46:50.581 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:50.948 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.6] 2024-09-17T23:46:50.949 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmkp65OhAA6CT6KX+G8GU3qxDJdfk6uA== 2024-09-17T23:46:50.949 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:50.949 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:50.949 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.6 2024-09-17T23:46:50.965 INFO:teuthology.run_tasks:Running task exec... 2024-09-17T23:46:50.974 INFO:teuthology.task.exec:Executing custom commands... 2024-09-17T23:46:50.974 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@smithi049.front.sepia.ceph.com 2024-09-17T23:46:50.975 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:51.338 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror] 2024-09-17T23:46:51.338 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQCdFOpmfZUbKhAABdKNXVO69OAlJyNDh+gWUQ== 2024-09-17T23:46:51.338 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror-peer" 2024-09-17T23:46:51.338 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:51.339 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror 2024-09-17T23:46:51.356 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:51.727 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.0] 2024-09-17T23:46:51.728 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQCdFOpmRvXQMBAAA4FZo1xEvCA4a7SPOTvrWg== 2024-09-17T23:46:51.728 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:51.728 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:51.728 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.0 2024-09-17T23:46:51.745 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:52.119 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.1] 2024-09-17T23:46:52.119 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQCdFOpmT0oyNhAAu934BeTq3SVoVsmdP7g4NQ== 2024-09-17T23:46:52.119 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:52.119 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:52.119 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.1 2024-09-17T23:46:52.137 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:52.498 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.2] 2024-09-17T23:46:52.498 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQCeFOpm788eABAAI9X0nomBDNFK8kbVqfVR0Q== 2024-09-17T23:46:52.498 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:52.498 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:52.499 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.2 2024-09-17T23:46:52.516 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:52.894 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.3] 2024-09-17T23:46:52.894 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQCeFOpmGNYHBRAAf0ZyH89sy5DaccfqeUcSaA== 2024-09-17T23:46:52.894 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:52.894 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:52.895 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.3 2024-09-17T23:46:52.912 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:53.271 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror] 2024-09-17T23:46:53.272 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmScDjFRAATNwzmXAGI5s/bBlVKDFRNQ== 2024-09-17T23:46:53.272 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror-peer" 2024-09-17T23:46:53.272 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:53.272 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror 2024-09-17T23:46:53.290 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:53.652 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.0] 2024-09-17T23:46:53.653 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmNYsnGxAAdzoj2zRP6aGtSPquqYl9sg== 2024-09-17T23:46:53.653 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:53.653 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:53.653 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.0 2024-09-17T23:46:53.670 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:54.027 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.1] 2024-09-17T23:46:54.028 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmxAJSIBAA3PJf/c5BfKF3ueQontpiLg== 2024-09-17T23:46:54.028 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:54.028 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:54.028 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.1 2024-09-17T23:46:54.045 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:54.397 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.2] 2024-09-17T23:46:54.397 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmY96LJRAAk/lDVjGCKbBKIUYIZxXjhw== 2024-09-17T23:46:54.397 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:54.397 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:54.397 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.2 2024-09-17T23:46:54.415 DEBUG:teuthology.orchestra.run.smithi049:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-17T23:46:54.791 INFO:teuthology.orchestra.run.smithi049.stdout:[client.mirror.3] 2024-09-17T23:46:54.792 INFO:teuthology.orchestra.run.smithi049.stdout: key = AQDDFOpmcQkKKxAAJhym19FFX+safELCaQD4PA== 2024-09-17T23:46:54.792 INFO:teuthology.orchestra.run.smithi049.stdout: caps mon = "profile rbd-mirror" 2024-09-17T23:46:54.792 INFO:teuthology.orchestra.run.smithi049.stdout: caps osd = "profile rbd" 2024-09-17T23:46:54.792 INFO:teuthology.orchestra.run.smithi049.stderr:updated caps for client.mirror.3 2024-09-17T23:46:54.811 INFO:teuthology.run_tasks:Running task workunit... 2024-09-17T23:46:54.837 INFO:tasks.workunit:Pulling workunits from ref b0d356b98e08f6bc59ef15652f0d0e4298fc83c7 2024-09-17T23:46:54.838 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-17T23:46:54.838 DEBUG:teuthology.orchestra.run.smithi049:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-17T23:46:54.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T23:46:54.843 INFO:teuthology.orchestra.run.smithi049.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2024-09-17T23:46:54.844 DEBUG:teuthology.orchestra.run.smithi049:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-17T23:46:54.891 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-17T23:46:54.892 DEBUG:teuthology.orchestra.run.smithi049:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2024-09-17T23:46:54.940 INFO:tasks.workunit:timeout=3h 2024-09-17T23:46:54.940 INFO:tasks.workunit:cleanup=True 2024-09-17T23:46:54.940 DEBUG:teuthology.orchestra.run.smithi049:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout b0d356b98e08f6bc59ef15652f0d0e4298fc83c7 2024-09-17T23:46:54.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2024-09-17T23:47:49.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:Updating files: 97% (12620/12893) Updating files: 98% (12636/12893) Updating files: 99% (12765/12893) Updating files: 100% (12893/12893) Updating files: 100% (12893/12893), done. 2024-09-17T23:47:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:Note: switching to 'b0d356b98e08f6bc59ef15652f0d0e4298fc83c7'. 2024-09-17T23:47:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2024-09-17T23:47:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-09-17T23:47:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:changes and commit them, and you can discard any commits you make in this 2024-09-17T23:47:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:state without impacting any branches by switching back to a branch. 2024-09-17T23:47:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2024-09-17T23:47:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:If you want to create a new branch to retain commits you create, you may 2024-09-17T23:47:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:do so (now or later) by using -c with the switch command. Example: 2024-09-17T23:47:50.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2024-09-17T23:47:50.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: git switch -c 2024-09-17T23:47:50.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2024-09-17T23:47:50.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:Or undo this operation with: 2024-09-17T23:47:50.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2024-09-17T23:47:50.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: git switch - 2024-09-17T23:47:50.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2024-09-17T23:47:50.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-09-17T23:47:50.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2024-09-17T23:47:50.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:HEAD is now at b0d356b98e0 Merge pull request #59811 from afreen23/wip-68079-squid 2024-09-17T23:47:50.790 DEBUG:teuthology.orchestra.run.smithi049:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2024-09-17T23:47:50.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-17T23:47:50.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2024-09-17T23:47:50.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-17T23:47:51.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-17T23:47:51.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-17T23:47:51.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2024-09-17T23:47:51.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2024-09-17T23:47:51.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-17T23:47:51.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2024-09-17T23:47:51.811 DEBUG:teuthology.orchestra.run.smithi049:> set -ex 2024-09-17T23:47:51.811 DEBUG:teuthology.orchestra.run.smithi049:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2024-09-17T23:47:51.857 INFO:tasks.workunit:Running workunits matching rbd/compare_mirror_images.sh on cluster1.client.mirror... 2024-09-17T23:47:51.858 INFO:tasks.workunit:Running workunit rbd/compare_mirror_images.sh... 2024-09-17T23:47:51.858 DEBUG:teuthology.orchestra.run.smithi049:workunit test rbd/compare_mirror_images.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b0d356b98e08f6bc59ef15652f0d0e4298fc83c7 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror RBD_DEVICE_TYPE=nbd adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/compare_mirror_images.sh 2024-09-17T23:47:51.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG_PREFIX=image-primary 2024-09-17T23:47:51.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MIRROR_IMAGE_MODE=snapshot 2024-09-17T23:47:51.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MIRROR_POOL_MODE=image 2024-09-17T23:47:51.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT_PREFIX=test-primary 2024-09-17T23:47:51.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RBD_IMAGE_FEATURES=layering,exclusive-lock,object-map,fast-diff 2024-09-17T23:47:51.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RBD_MIRROR_INSTANCES=1 2024-09-17T23:47:51.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RBD_MIRROR_MODE=snapshot 2024-09-17T23:47:51.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RBD_MIRROR_USE_EXISTING_CLUSTER=1 2024-09-17T23:47:51.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/compare_mirror_images.sh 2024-09-17T23:47:51.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2024-09-17T23:47:51.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ type xmlstarlet 2024-09-17T23:47:51.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ XMLSTARLET=xmlstarlet 2024-09-17T23:47:51.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ RBD_MIRROR_INSTANCES=1 2024-09-17T23:47:51.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CLUSTER1=cluster1 2024-09-17T23:47:51.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CLUSTER2=cluster2 2024-09-17T23:47:51.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ PEER_CLUSTER_SUFFIX= 2024-09-17T23:47:51.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ POOL=mirror 2024-09-17T23:47:51.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ PARENT_POOL=mirror_parent 2024-09-17T23:47:51.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ NS1=ns1 2024-09-17T23:47:51.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ NS2=ns2 2024-09-17T23:47:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ TEMPDIR= 2024-09-17T23:47:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ID=mirror 2024-09-17T23:47:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,object-map,fast-diff 2024-09-17T23:47:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2024-09-17T23:47:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ RBD_MIRROR_MODE=snapshot 2024-09-17T23:47:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ MIRROR_POOL_MODE=image 2024-09-17T23:47:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ '[' snapshot = snapshot ']' 2024-09-17T23:47:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ MIRROR_POOL_MODE=image 2024-09-17T23:47:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ export 'CEPH_ARGS=--id mirror' 2024-09-17T23:47:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS='--id mirror' 2024-09-17T23:47:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ LAST_MIRROR_INSTANCE=0 2024-09-17T23:47:51.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/compare_mirror_images.sh 2024-09-17T23:47:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2024-09-17T23:47:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2024-09-17T23:47:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_BIN=. 2024-09-17T23:47:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_SRC=. 2024-09-17T23:47:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ '[' -e CMakeCache.txt ']' 2024-09-17T23:47:51.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ '[' 0 -gt 0 ']' 2024-09-17T23:47:51.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ setup 2024-09-17T23:47:51.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local c 2024-09-17T23:47:51.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ trap 'cleanup $?' INT TERM EXIT 2024-09-17T23:47:51.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ setup_tempdir 2024-09-17T23:47:51.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:47:51.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ mktemp -d 2024-09-17T23:47:51.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ TEMPDIR=/tmp/tmp.Ir77rubWei 2024-09-17T23:47:51.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -z 1 ']' 2024-09-17T23:47:51.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ setup_pools cluster1 cluster2 2024-09-17T23:47:51.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:47:51.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local remote_cluster=cluster2 2024-09-17T23:47:51.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mon_map_file 2024-09-17T23:47:51.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mon_addr 2024-09-17T23:47:51.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local admin_key_file 2024-09-17T23:47:51.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local uuid 2024-09-17T23:47:51.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-17T23:47:51.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2024-09-17T23:47:52.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:pool 'mirror' already exists 2024-09-17T23:47:52.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-17T23:47:52.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2024-09-17T23:47:53.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:pool 'mirror_parent' already exists 2024-09-17T23:47:53.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-17T23:47:53.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 pool init mirror 2024-09-17T23:47:56.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-17T23:47:56.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2024-09-17T23:47:59.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:47:59.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-17T23:47:59.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror image 2024-09-17T23:47:59.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2024-09-17T23:47:59.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2024-09-17T23:48:00.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2024-09-17T23:48:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2024-09-17T23:48:00.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2024-09-17T23:48:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -z ']' 2024-09-17T23:48:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -z ']' 2024-09-17T23:48:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ peer_add cluster1 mirror cluster2 2024-09-17T23:48:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local client_cluster=cluster2 2024-09-17T23:48:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:00.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local remote_cluster=cluster2 2024-09-17T23:48:00.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local uuid_var_name 2024-09-17T23:48:00.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:48:00.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local error_code 2024-09-17T23:48:00.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local peer_uuid 2024-09-17T23:48:00.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 16 32 2024-09-17T23:48:00.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ set +e 2024-09-17T23:48:00.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2024-09-17T23:48:00.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ peer_uuid=e614ab96-8e11-46be-8c81-ea65aeeb88dc 2024-09-17T23:48:00.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ error_code=0 2024-09-17T23:48:00.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ set -e 2024-09-17T23:48:00.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' 0 -eq 17 ']' 2024-09-17T23:48:00.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test 0 -eq 0 2024-09-17T23:48:00.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:48:00.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ peer_add cluster1 mirror_parent cluster2 2024-09-17T23:48:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror_parent 2024-09-17T23:48:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local client_cluster=cluster2 2024-09-17T23:48:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local remote_cluster=cluster2 2024-09-17T23:48:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local uuid_var_name 2024-09-17T23:48:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:48:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local error_code 2024-09-17T23:48:00.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local peer_uuid 2024-09-17T23:48:00.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 16 32 2024-09-17T23:48:00.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ set +e 2024-09-17T23:48:00.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2024-09-17T23:48:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ peer_uuid=c372f876-16f5-42af-b8e1-81c5fb97b2ae 2024-09-17T23:48:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ error_code=0 2024-09-17T23:48:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ set -e 2024-09-17T23:48:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' 0 -eq 17 ']' 2024-09-17T23:48:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test 0 -eq 0 2024-09-17T23:48:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:48:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ setup_pools cluster2 cluster1 2024-09-17T23:48:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:48:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local remote_cluster=cluster1 2024-09-17T23:48:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mon_map_file 2024-09-17T23:48:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mon_addr 2024-09-17T23:48:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local admin_key_file 2024-09-17T23:48:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local uuid 2024-09-17T23:48:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-17T23:48:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2024-09-17T23:48:01.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:pool 'mirror' already exists 2024-09-17T23:48:01.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-17T23:48:01.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2024-09-17T23:48:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:pool 'mirror_parent' already exists 2024-09-17T23:48:02.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-17T23:48:02.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster2 pool init mirror 2024-09-17T23:48:05.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-17T23:48:05.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2024-09-17T23:48:08.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:48:08.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-17T23:48:08.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror image 2024-09-17T23:48:08.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2024-09-17T23:48:08.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2024-09-17T23:48:08.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2024-09-17T23:48:08.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2024-09-17T23:48:08.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2024-09-17T23:48:08.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -z ']' 2024-09-17T23:48:08.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -z ']' 2024-09-17T23:48:08.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ peer_add cluster2 mirror cluster1 2024-09-17T23:48:08.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:48:08.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:08.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:08.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:08.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local client_cluster=cluster1 2024-09-17T23:48:08.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:08.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local remote_cluster=cluster1 2024-09-17T23:48:08.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local uuid_var_name 2024-09-17T23:48:08.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:48:08.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local error_code 2024-09-17T23:48:08.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local peer_uuid 2024-09-17T23:48:08.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 16 32 2024-09-17T23:48:08.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ set +e 2024-09-17T23:48:08.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2024-09-17T23:48:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ peer_uuid=4f77d5df-807f-4e80-9f65-00f8e5d081e4 2024-09-17T23:48:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ error_code=0 2024-09-17T23:48:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ set -e 2024-09-17T23:48:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' 0 -eq 17 ']' 2024-09-17T23:48:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test 0 -eq 0 2024-09-17T23:48:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:48:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ peer_add cluster2 mirror_parent cluster1 2024-09-17T23:48:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:48:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror_parent 2024-09-17T23:48:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local client_cluster=cluster1 2024-09-17T23:48:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local remote_cluster=cluster1 2024-09-17T23:48:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local uuid_var_name 2024-09-17T23:48:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:48:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local error_code 2024-09-17T23:48:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local peer_uuid 2024-09-17T23:48:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 16 32 2024-09-17T23:48:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ set +e 2024-09-17T23:48:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2024-09-17T23:48:08.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ peer_uuid=fbe24f48-3e03-4bdb-8f2f-1cdcc0c89cb5 2024-09-17T23:48:08.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ error_code=0 2024-09-17T23:48:08.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ set -e 2024-09-17T23:48:08.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' 0 -eq 17 ']' 2024-09-17T23:48:08.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test 0 -eq 0 2024-09-17T23:48:08.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:48:08.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:08.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:48:08.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ start_mirrors cluster1 2024-09-17T23:48:08.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:08.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ seq 0 0 2024-09-17T23:48:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-17T23:48:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ start_mirror cluster1:0 2024-09-17T23:48:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1:0 2024-09-17T23:48:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local instance 2024-09-17T23:48:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-17T23:48:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local val=cluster1:0 2024-09-17T23:48:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster_var_name=cluster 2024-09-17T23:48:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local instance_var_name=instance 2024-09-17T23:48:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cluster=cluster1 2024-09-17T23:48:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ instance=0 2024-09-17T23:48:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-17T23:48:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ eval cluster=cluster1 2024-09-17T23:48:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ cluster=cluster1 2024-09-17T23:48:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ eval instance=0 2024-09-17T23:48:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ instance=0 2024-09-17T23:48:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test -n '' 2024-09-17T23:48:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2024-09-17T23:48:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ start_mirrors cluster2 2024-09-17T23:48:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:48:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ seq 0 0 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ start_mirror cluster2:0 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2:0 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local instance 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ set_cluster_instance cluster2:0 cluster instance 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local val=cluster2:0 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster_var_name=cluster 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local instance_var_name=instance 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cluster=cluster2 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ instance=0 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' 0 = cluster2:0 ']' 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ eval cluster=cluster2 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ cluster=cluster2 2024-09-17T23:48:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ eval instance=0 2024-09-17T23:48:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ instance=0 2024-09-17T23:48:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test -n '' 2024-09-17T23:48:08.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2024-09-17T23:48:08.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wget https://download.ceph.com/qa/linux-5.4.tar.gz 2024-09-17T23:48:08.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:--2024-09-17 23:48:08-- https://download.ceph.com/qa/linux-5.4.tar.gz 2024-09-17T23:48:09.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:Resolving download.ceph.com (download.ceph.com)... 158.69.68.124, 2607:5300:201:2000::3:58a1 2024-09-17T23:48:09.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:Connecting to download.ceph.com (download.ceph.com)|158.69.68.124|:443... connected. 2024-09-17T23:48:09.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:HTTP request sent, awaiting response... 200 OK 2024-09-17T23:48:09.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:Length: 172616875 (165M) [application/octet-stream] 2024-09-17T23:48:09.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:Saving to: 'linux-5.4.tar.gz' 2024-09-17T23:48:09.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2024-09-17T23:48:09.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 0K .......... .......... .......... .......... .......... 0% 2.40M 69s 2024-09-17T23:48:09.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50K .......... .......... .......... .......... .......... 0% 2.40M 69s 2024-09-17T23:48:09.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 100K .......... .......... .......... .......... .......... 0% 114M 46s 2024-09-17T23:48:09.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 150K .......... .......... .......... .......... .......... 0% 2.45M 51s 2024-09-17T23:48:09.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 200K .......... .......... .......... .......... .......... 0% 107M 41s 2024-09-17T23:48:09.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 250K .......... .......... .......... .......... .......... 0% 381M 35s 2024-09-17T23:48:09.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 300K .......... .......... .......... .......... .......... 0% 371M 30s 2024-09-17T23:48:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 350K .......... .......... .......... .......... .......... 0% 2.51M 34s 2024-09-17T23:48:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 400K .......... .......... .......... .......... .......... 0% 291M 30s 2024-09-17T23:48:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 450K .......... .......... .......... .......... .......... 0% 122M 28s 2024-09-17T23:48:09.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 500K .......... .......... .......... .......... .......... 0% 267M 25s 2024-09-17T23:48:09.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 550K .......... .......... .......... .......... .......... 0% 301M 23s 2024-09-17T23:48:09.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 600K .......... .......... .......... .......... .......... 0% 406M 21s 2024-09-17T23:48:09.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 650K .......... .......... .......... .......... .......... 0% 62.3M 20s 2024-09-17T23:48:09.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 700K .......... .......... .......... .......... .......... 0% 56.6M 19s 2024-09-17T23:48:09.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 750K .......... .......... .......... .......... .......... 0% 2.75M 21s 2024-09-17T23:48:09.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 800K .......... .......... .......... .......... .......... 0% 265M 20s 2024-09-17T23:48:09.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 850K .......... .......... .......... .......... .......... 0% 394M 19s 2024-09-17T23:48:09.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 900K .......... .......... .......... .......... .......... 0% 404M 18s 2024-09-17T23:48:09.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 950K .......... .......... .......... .......... .......... 0% 344M 17s 2024-09-17T23:48:09.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1000K .......... .......... .......... .......... .......... 0% 399M 16s 2024-09-17T23:48:09.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1050K .......... .......... .......... .......... .......... 0% 53.0M 16s 2024-09-17T23:48:09.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1100K .......... .......... .......... .......... .......... 0% 21.8M 15s 2024-09-17T23:48:09.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1150K .......... .......... .......... .......... .......... 0% 22.7M 15s 2024-09-17T23:48:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1200K .......... .......... .......... .......... .......... 0% 24.3M 15s 2024-09-17T23:48:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1250K .......... .......... .......... .......... .......... 0% 247M 14s 2024-09-17T23:48:09.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1300K .......... .......... .......... .......... .......... 0% 31.7M 14s 2024-09-17T23:48:09.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1350K .......... .......... .......... .......... .......... 0% 26.1M 14s 2024-09-17T23:48:09.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1400K .......... .......... .......... .......... .......... 0% 23.7M 13s 2024-09-17T23:48:09.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1450K .......... .......... .......... .......... .......... 0% 58.4M 13s 2024-09-17T23:48:09.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1500K .......... .......... .......... .......... .......... 0% 28.1M 13s 2024-09-17T23:48:09.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1550K .......... .......... .......... .......... .......... 0% 11.9M 13s 2024-09-17T23:48:09.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1600K .......... .......... .......... .......... .......... 0% 352M 12s 2024-09-17T23:48:09.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1650K .......... .......... .......... .......... .......... 1% 23.3M 12s 2024-09-17T23:48:09.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1700K .......... .......... .......... .......... .......... 1% 25.5M 12s 2024-09-17T23:48:09.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1750K .......... .......... .......... .......... .......... 1% 216M 12s 2024-09-17T23:48:09.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1800K .......... .......... .......... .......... .......... 1% 23.6M 12s 2024-09-17T23:48:09.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1850K .......... .......... .......... .......... .......... 1% 22.1M 11s 2024-09-17T23:48:09.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1900K .......... .......... .......... .......... .......... 1% 24.3M 11s 2024-09-17T23:48:09.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 1950K .......... .......... .......... .......... .......... 1% 24.5M 11s 2024-09-17T23:48:09.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2000K .......... .......... .......... .......... .......... 1% 299M 11s 2024-09-17T23:48:09.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2050K .......... .......... .......... .......... .......... 1% 23.1M 11s 2024-09-17T23:48:09.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2100K .......... .......... .......... .......... .......... 1% 23.8M 11s 2024-09-17T23:48:09.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2150K .......... .......... .......... .......... .......... 1% 23.0M 11s 2024-09-17T23:48:09.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2200K .......... .......... .......... .......... .......... 1% 54.4M 11s 2024-09-17T23:48:09.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2250K .......... .......... .......... .......... .......... 1% 42.0M 10s 2024-09-17T23:48:09.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2300K .......... .......... .......... .......... .......... 1% 22.3M 10s 2024-09-17T23:48:09.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2350K .......... .......... .......... .......... .......... 1% 37.7M 10s 2024-09-17T23:48:09.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2400K .......... .......... .......... .......... .......... 1% 17.5M 10s 2024-09-17T23:48:09.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2450K .......... .......... .......... .......... .......... 1% 33.7M 10s 2024-09-17T23:48:09.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2500K .......... .......... .......... .......... .......... 1% 73.4M 10s 2024-09-17T23:48:09.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2550K .......... .......... .......... .......... .......... 1% 19.9M 10s 2024-09-17T23:48:09.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2600K .......... .......... .......... .......... .......... 1% 28.0M 10s 2024-09-17T23:48:09.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2650K .......... .......... .......... .......... .......... 1% 19.9M 10s 2024-09-17T23:48:09.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2700K .......... .......... .......... .......... .......... 1% 119M 10s 2024-09-17T23:48:09.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2750K .......... .......... .......... .......... .......... 1% 18.3M 10s 2024-09-17T23:48:09.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2800K .......... .......... .......... .......... .......... 1% 65.9M 9s 2024-09-17T23:48:09.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2850K .......... .......... .......... .......... .......... 1% 31.7M 9s 2024-09-17T23:48:09.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2900K .......... .......... .......... .......... .......... 1% 26.9M 9s 2024-09-17T23:48:09.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2950K .......... .......... .......... .......... .......... 1% 16.0M 9s 2024-09-17T23:48:09.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3000K .......... .......... .......... .......... .......... 1% 144M 9s 2024-09-17T23:48:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3050K .......... .......... .......... .......... .......... 1% 27.2M 9s 2024-09-17T23:48:09.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3100K .......... .......... .......... .......... .......... 1% 22.1M 9s 2024-09-17T23:48:09.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3150K .......... .......... .......... .......... .......... 1% 25.4M 9s 2024-09-17T23:48:09.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3200K .......... .......... .......... .......... .......... 1% 12.7M 9s 2024-09-17T23:48:09.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3250K .......... .......... .......... .......... .......... 1% 121M 9s 2024-09-17T23:48:09.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3300K .......... .......... .......... .......... .......... 1% 93.4M 9s 2024-09-17T23:48:09.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3350K .......... .......... .......... .......... .......... 2% 30.1M 9s 2024-09-17T23:48:09.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3400K .......... .......... .......... .......... .......... 2% 25.5M 9s 2024-09-17T23:48:09.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3450K .......... .......... .......... .......... .......... 2% 62.3M 9s 2024-09-17T23:48:09.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3500K .......... .......... .......... .......... .......... 2% 33.6M 9s 2024-09-17T23:48:09.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3550K .......... .......... .......... .......... .......... 2% 23.5M 9s 2024-09-17T23:48:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3600K .......... .......... .......... .......... .......... 2% 25.4M 9s 2024-09-17T23:48:09.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3650K .......... .......... .......... .......... .......... 2% 28.5M 9s 2024-09-17T23:48:09.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3700K .......... .......... .......... .......... .......... 2% 35.4M 8s 2024-09-17T23:48:09.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3750K .......... .......... .......... .......... .......... 2% 37.4M 8s 2024-09-17T23:48:09.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3800K .......... .......... .......... .......... .......... 2% 23.5M 8s 2024-09-17T23:48:09.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3850K .......... .......... .......... .......... .......... 2% 34.5M 8s 2024-09-17T23:48:09.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3900K .......... .......... .......... .......... .......... 2% 22.7M 8s 2024-09-17T23:48:09.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 3950K .......... .......... .......... .......... .......... 2% 24.8M 8s 2024-09-17T23:48:09.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4000K .......... .......... .......... .......... .......... 2% 25.6M 8s 2024-09-17T23:48:09.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4050K .......... .......... .......... .......... .......... 2% 79.4M 8s 2024-09-17T23:48:09.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4100K .......... .......... .......... .......... .......... 2% 29.3M 8s 2024-09-17T23:48:09.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4150K .......... .......... .......... .......... .......... 2% 25.1M 8s 2024-09-17T23:48:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4200K .......... .......... .......... .......... .......... 2% 24.6M 8s 2024-09-17T23:48:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4250K .......... .......... .......... .......... .......... 2% 47.3M 8s 2024-09-17T23:48:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4300K .......... .......... .......... .......... .......... 2% 27.1M 8s 2024-09-17T23:48:09.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4350K .......... .......... .......... .......... .......... 2% 24.0M 8s 2024-09-17T23:48:09.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4400K .......... .......... .......... .......... .......... 2% 33.2M 8s 2024-09-17T23:48:09.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4450K .......... .......... .......... .......... .......... 2% 23.1M 8s 2024-09-17T23:48:09.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4500K .......... .......... .......... .......... .......... 2% 26.1M 8s 2024-09-17T23:48:09.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4550K .......... .......... .......... .......... .......... 2% 56.6M 8s 2024-09-17T23:48:09.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4600K .......... .......... .......... .......... .......... 2% 27.4M 8s 2024-09-17T23:48:09.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4650K .......... .......... .......... .......... .......... 2% 27.6M 8s 2024-09-17T23:48:09.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4700K .......... .......... .......... .......... .......... 2% 26.2M 8s 2024-09-17T23:48:09.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4750K .......... .......... .......... .......... .......... 2% 23.2M 8s 2024-09-17T23:48:09.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4800K .......... .......... .......... .......... .......... 2% 48.9M 8s 2024-09-17T23:48:09.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4850K .......... .......... .......... .......... .......... 2% 24.7M 8s 2024-09-17T23:48:09.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4900K .......... .......... .......... .......... .......... 2% 39.7M 8s 2024-09-17T23:48:09.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 4950K .......... .......... .......... .......... .......... 2% 23.3M 8s 2024-09-17T23:48:09.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5000K .......... .......... .......... .......... .......... 2% 22.7M 8s 2024-09-17T23:48:09.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5050K .......... .......... .......... .......... .......... 3% 56.6M 8s 2024-09-17T23:48:09.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5100K .......... .......... .......... .......... .......... 3% 33.4M 8s 2024-09-17T23:48:09.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5150K .......... .......... .......... .......... .......... 3% 24.8M 8s 2024-09-17T23:48:09.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5200K .......... .......... .......... .......... .......... 3% 24.8M 8s 2024-09-17T23:48:09.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5250K .......... .......... .......... .......... .......... 3% 24.2M 8s 2024-09-17T23:48:09.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5300K .......... .......... .......... .......... .......... 3% 48.8M 8s 2024-09-17T23:48:09.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5350K .......... .......... .......... .......... .......... 3% 23.0M 7s 2024-09-17T23:48:09.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5400K .......... .......... .......... .......... .......... 3% 27.0M 7s 2024-09-17T23:48:09.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5450K .......... .......... .......... .......... .......... 3% 32.4M 7s 2024-09-17T23:48:09.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5500K .......... .......... .......... .......... .......... 3% 50.2M 7s 2024-09-17T23:48:09.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5550K .......... .......... .......... .......... .......... 3% 15.6M 7s 2024-09-17T23:48:09.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5600K .......... .......... .......... .......... .......... 3% 55.7M 7s 2024-09-17T23:48:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5650K .......... .......... .......... .......... .......... 3% 32.1M 7s 2024-09-17T23:48:09.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5700K .......... .......... .......... .......... .......... 3% 24.2M 7s 2024-09-17T23:48:09.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5750K .......... .......... .......... .......... .......... 3% 25.3M 7s 2024-09-17T23:48:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5800K .......... .......... .......... .......... .......... 3% 53.8M 7s 2024-09-17T23:48:09.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5850K .......... .......... .......... .......... .......... 3% 27.6M 7s 2024-09-17T23:48:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5900K .......... .......... .......... .......... .......... 3% 31.7M 7s 2024-09-17T23:48:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 5950K .......... .......... .......... .......... .......... 3% 23.2M 7s 2024-09-17T23:48:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6000K .......... .......... .......... .......... .......... 3% 24.5M 7s 2024-09-17T23:48:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6050K .......... .......... .......... .......... .......... 3% 52.7M 7s 2024-09-17T23:48:09.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6100K .......... .......... .......... .......... .......... 3% 22.4M 7s 2024-09-17T23:48:09.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6150K .......... .......... .......... .......... .......... 3% 34.3M 7s 2024-09-17T23:48:09.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6200K .......... .......... .......... .......... .......... 3% 29.8M 7s 2024-09-17T23:48:09.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6250K .......... .......... .......... .......... .......... 3% 22.2M 7s 2024-09-17T23:48:09.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6300K .......... .......... .......... .......... .......... 3% 49.2M 7s 2024-09-17T23:48:09.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6350K .......... .......... .......... .......... .......... 3% 23.2M 7s 2024-09-17T23:48:09.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6400K .......... .......... .......... .......... .......... 3% 25.2M 7s 2024-09-17T23:48:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6450K .......... .......... .......... .......... .......... 3% 27.2M 7s 2024-09-17T23:48:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6500K .......... .......... .......... .......... .......... 3% 33.0M 7s 2024-09-17T23:48:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6550K .......... .......... .......... .......... .......... 3% 47.5M 7s 2024-09-17T23:48:09.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6600K .......... .......... .......... .......... .......... 3% 24.6M 7s 2024-09-17T23:48:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6650K .......... .......... .......... .......... .......... 3% 39.4M 7s 2024-09-17T23:48:09.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6700K .......... .......... .......... .......... .......... 4% 21.9M 7s 2024-09-17T23:48:09.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6750K .......... .......... .......... .......... .......... 4% 23.0M 7s 2024-09-17T23:48:09.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6800K .......... .......... .......... .......... .......... 4% 25.4M 7s 2024-09-17T23:48:09.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6850K .......... .......... .......... .......... .......... 4% 72.3M 7s 2024-09-17T23:48:09.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6900K .......... .......... .......... .......... .......... 4% 23.9M 7s 2024-09-17T23:48:09.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 6950K .......... .......... .......... .......... .......... 4% 30.3M 7s 2024-09-17T23:48:09.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7000K .......... .......... .......... .......... .......... 4% 24.0M 7s 2024-09-17T23:48:09.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7050K .......... .......... .......... .......... .......... 4% 63.6M 7s 2024-09-17T23:48:09.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7100K .......... .......... .......... .......... .......... 4% 25.4M 7s 2024-09-17T23:48:09.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7150K .......... .......... .......... .......... .......... 4% 22.1M 7s 2024-09-17T23:48:09.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7200K .......... .......... .......... .......... .......... 4% 34.4M 7s 2024-09-17T23:48:09.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7250K .......... .......... .......... .......... .......... 4% 23.3M 7s 2024-09-17T23:48:09.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7300K .......... .......... .......... .......... .......... 4% 25.3M 7s 2024-09-17T23:48:09.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7350K .......... .......... .......... .......... .......... 4% 57.6M 7s 2024-09-17T23:48:09.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7400K .......... .......... .......... .......... .......... 4% 30.1M 7s 2024-09-17T23:48:09.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7450K .......... .......... .......... .......... .......... 4% 26.4M 7s 2024-09-17T23:48:09.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7500K .......... .......... .......... .......... .......... 4% 47.8M 7s 2024-09-17T23:48:09.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7550K .......... .......... .......... .......... .......... 4% 22.0M 7s 2024-09-17T23:48:09.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7600K .......... .......... .......... .......... .......... 4% 30.7M 7s 2024-09-17T23:48:09.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7650K .......... .......... .......... .......... .......... 4% 23.2M 7s 2024-09-17T23:48:09.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7700K .......... .......... .......... .......... .......... 4% 34.9M 7s 2024-09-17T23:48:09.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7750K .......... .......... .......... .......... .......... 4% 30.2M 7s 2024-09-17T23:48:09.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7800K .......... .......... .......... .......... .......... 4% 19.5M 7s 2024-09-17T23:48:09.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7850K .......... .......... .......... .......... .......... 4% 56.9M 7s 2024-09-17T23:48:09.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7900K .......... .......... .......... .......... .......... 4% 31.4M 7s 2024-09-17T23:48:09.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 7950K .......... .......... .......... .......... .......... 4% 25.5M 7s 2024-09-17T23:48:09.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8000K .......... .......... .......... .......... .......... 4% 25.2M 7s 2024-09-17T23:48:09.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8050K .......... .......... .......... .......... .......... 4% 23.5M 7s 2024-09-17T23:48:09.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8100K .......... .......... .......... .......... .......... 4% 48.5M 7s 2024-09-17T23:48:09.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8150K .......... .......... .......... .......... .......... 4% 37.2M 7s 2024-09-17T23:48:09.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8200K .......... .......... .......... .......... .......... 4% 24.6M 7s 2024-09-17T23:48:09.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8250K .......... .......... .......... .......... .......... 4% 24.7M 7s 2024-09-17T23:48:09.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8300K .......... .......... .......... .......... .......... 4% 58.0M 7s 2024-09-17T23:48:09.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8350K .......... .......... .......... .......... .......... 4% 23.4M 7s 2024-09-17T23:48:09.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8400K .......... .......... .......... .......... .......... 5% 23.5M 7s 2024-09-17T23:48:09.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8450K .......... .......... .......... .......... .......... 5% 34.6M 7s 2024-09-17T23:48:09.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8500K .......... .......... .......... .......... .......... 5% 26.0M 7s 2024-09-17T23:48:09.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8550K .......... .......... .......... .......... .......... 5% 23.6M 7s 2024-09-17T23:48:09.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8600K .......... .......... .......... .......... .......... 5% 49.1M 7s 2024-09-17T23:48:09.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8650K .......... .......... .......... .......... .......... 5% 24.5M 7s 2024-09-17T23:48:09.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8700K .......... .......... .......... .......... .......... 5% 32.2M 7s 2024-09-17T23:48:09.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8750K .......... .......... .......... .......... .......... 5% 24.1M 7s 2024-09-17T23:48:09.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8800K .......... .......... .......... .......... .......... 5% 25.0M 7s 2024-09-17T23:48:09.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8850K .......... .......... .......... .......... .......... 5% 47.2M 7s 2024-09-17T23:48:09.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8900K .......... .......... .......... .......... .......... 5% 25.1M 7s 2024-09-17T23:48:09.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 8950K .......... .......... .......... .......... .......... 5% 35.4M 7s 2024-09-17T23:48:09.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9000K .......... .......... .......... .......... .......... 5% 24.3M 7s 2024-09-17T23:48:09.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9050K .......... .......... .......... .......... .......... 5% 24.6M 7s 2024-09-17T23:48:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9100K .......... .......... .......... .......... .......... 5% 55.1M 7s 2024-09-17T23:48:09.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9150K .......... .......... .......... .......... .......... 5% 23.3M 7s 2024-09-17T23:48:09.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9200K .......... .......... .......... .......... .......... 5% 24.6M 7s 2024-09-17T23:48:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9250K .......... .......... .......... .......... .......... 5% 35.2M 6s 2024-09-17T23:48:09.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9300K .......... .......... .......... .......... .......... 5% 23.8M 6s 2024-09-17T23:48:09.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9350K .......... .......... .......... .......... .......... 5% 51.0M 6s 2024-09-17T23:48:09.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9400K .......... .......... .......... .......... .......... 5% 24.3M 6s 2024-09-17T23:48:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9450K .......... .......... .......... .......... .......... 5% 36.3M 6s 2024-09-17T23:48:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9500K .......... .......... .......... .......... .......... 5% 24.4M 6s 2024-09-17T23:48:09.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9550K .......... .......... .......... .......... .......... 5% 23.5M 6s 2024-09-17T23:48:09.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9600K .......... .......... .......... .......... .......... 5% 23.8M 6s 2024-09-17T23:48:09.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9650K .......... .......... .......... .......... .......... 5% 57.9M 6s 2024-09-17T23:48:09.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9700K .......... .......... .......... .......... .......... 5% 23.4M 6s 2024-09-17T23:48:09.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9750K .......... .......... .......... .......... .......... 5% 34.4M 6s 2024-09-17T23:48:09.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9800K .......... .......... .......... .......... .......... 5% 25.6M 6s 2024-09-17T23:48:09.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9850K .......... .......... .......... .......... .......... 5% 48.6M 6s 2024-09-17T23:48:09.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9900K .......... .......... .......... .......... .......... 5% 25.1M 6s 2024-09-17T23:48:09.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 9950K .......... .......... .......... .......... .......... 5% 23.7M 6s 2024-09-17T23:48:09.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10000K .......... .......... .......... .......... .......... 5% 36.6M 6s 2024-09-17T23:48:09.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10050K .......... .......... .......... .......... .......... 5% 24.7M 6s 2024-09-17T23:48:09.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10100K .......... .......... .......... .......... .......... 6% 21.5M 6s 2024-09-17T23:48:09.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10150K .......... .......... .......... .......... .......... 6% 65.7M 6s 2024-09-17T23:48:09.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10200K .......... .......... .......... .......... .......... 6% 23.3M 6s 2024-09-17T23:48:09.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10250K .......... .......... .......... .......... .......... 6% 50.6M 6s 2024-09-17T23:48:09.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10300K .......... .......... .......... .......... .......... 6% 20.9M 6s 2024-09-17T23:48:09.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10350K .......... .......... .......... .......... .......... 6% 23.8M 6s 2024-09-17T23:48:09.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10400K .......... .......... .......... .......... .......... 6% 48.0M 6s 2024-09-17T23:48:09.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10450K .......... .......... .......... .......... .......... 6% 25.7M 6s 2024-09-17T23:48:09.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10500K .......... .......... .......... .......... .......... 6% 22.7M 6s 2024-09-17T23:48:09.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10550K .......... .......... .......... .......... .......... 6% 39.3M 6s 2024-09-17T23:48:09.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10600K .......... .......... .......... .......... .......... 6% 47.8M 6s 2024-09-17T23:48:09.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10650K .......... .......... .......... .......... .......... 6% 24.9M 6s 2024-09-17T23:48:09.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10700K .......... .......... .......... .......... .......... 6% 32.5M 6s 2024-09-17T23:48:09.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10750K .......... .......... .......... .......... .......... 6% 23.4M 6s 2024-09-17T23:48:09.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10800K .......... .......... .......... .......... .......... 6% 25.2M 6s 2024-09-17T23:48:09.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10850K .......... .......... .......... .......... .......... 6% 25.9M 6s 2024-09-17T23:48:09.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10900K .......... .......... .......... .......... .......... 6% 56.9M 6s 2024-09-17T23:48:09.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 10950K .......... .......... .......... .......... .......... 6% 30.7M 6s 2024-09-17T23:48:09.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11000K .......... .......... .......... .......... .......... 6% 25.9M 6s 2024-09-17T23:48:09.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11050K .......... .......... .......... .......... .......... 6% 23.0M 6s 2024-09-17T23:48:09.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11100K .......... .......... .......... .......... .......... 6% 56.2M 6s 2024-09-17T23:48:09.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11150K .......... .......... .......... .......... .......... 6% 23.5M 6s 2024-09-17T23:48:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11200K .......... .......... .......... .......... .......... 6% 24.9M 6s 2024-09-17T23:48:09.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11250K .......... .......... .......... .......... .......... 6% 33.9M 6s 2024-09-17T23:48:09.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11300K .......... .......... .......... .......... .......... 6% 24.5M 6s 2024-09-17T23:48:09.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11350K .......... .......... .......... .......... .......... 6% 26.9M 6s 2024-09-17T23:48:09.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11400K .......... .......... .......... .......... .......... 6% 42.1M 6s 2024-09-17T23:48:09.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11450K .......... .......... .......... .......... .......... 6% 24.1M 6s 2024-09-17T23:48:09.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11500K .......... .......... .......... .......... .......... 6% 36.6M 6s 2024-09-17T23:48:09.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11550K .......... .......... .......... .......... .......... 6% 23.8M 6s 2024-09-17T23:48:09.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11600K .......... .......... .......... .......... .......... 6% 27.3M 6s 2024-09-17T23:48:09.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11650K .......... .......... .......... .......... .......... 6% 41.5M 6s 2024-09-17T23:48:09.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11700K .......... .......... .......... .......... .......... 6% 25.4M 6s 2024-09-17T23:48:09.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11750K .......... .......... .......... .......... .......... 7% 22.9M 6s 2024-09-17T23:48:09.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11800K .......... .......... .......... .......... .......... 7% 34.7M 6s 2024-09-17T23:48:09.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11850K .......... .......... .......... .......... .......... 7% 25.8M 6s 2024-09-17T23:48:09.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11900K .......... .......... .......... .......... .......... 7% 58.4M 6s 2024-09-17T23:48:09.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 11950K .......... .......... .......... .......... .......... 7% 23.6M 6s 2024-09-17T23:48:09.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12000K .......... .......... .......... .......... .......... 7% 23.9M 6s 2024-09-17T23:48:09.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12050K .......... .......... .......... .......... .......... 7% 32.9M 6s 2024-09-17T23:48:09.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12100K .......... .......... .......... .......... .......... 7% 32.4M 6s 2024-09-17T23:48:09.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12150K .......... .......... .......... .......... .......... 7% 31.5M 6s 2024-09-17T23:48:09.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12200K .......... .......... .......... .......... .......... 7% 36.3M 6s 2024-09-17T23:48:09.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12250K .......... .......... .......... .......... .......... 7% 25.7M 6s 2024-09-17T23:48:09.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12300K .......... .......... .......... .......... .......... 7% 24.8M 6s 2024-09-17T23:48:09.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12350K .......... .......... .......... .......... .......... 7% 28.1M 6s 2024-09-17T23:48:09.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12400K .......... .......... .......... .......... .......... 7% 19.8M 6s 2024-09-17T23:48:09.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12450K .......... .......... .......... .......... .......... 7% 79.9M 6s 2024-09-17T23:48:09.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12500K .......... .......... .......... .......... .......... 7% 26.9M 6s 2024-09-17T23:48:09.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12550K .......... .......... .......... .......... .......... 7% 24.4M 6s 2024-09-17T23:48:09.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12600K .......... .......... .......... .......... .......... 7% 25.6M 6s 2024-09-17T23:48:09.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12650K .......... .......... .......... .......... .......... 7% 47.7M 6s 2024-09-17T23:48:09.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12700K .......... .......... .......... .......... .......... 7% 27.2M 6s 2024-09-17T23:48:09.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12750K .......... .......... .......... .......... .......... 7% 25.1M 6s 2024-09-17T23:48:09.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12800K .......... .......... .......... .......... .......... 7% 24.4M 6s 2024-09-17T23:48:09.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12850K .......... .......... .......... .......... .......... 7% 70.8M 6s 2024-09-17T23:48:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12900K .......... .......... .......... .......... .......... 7% 16.7M 6s 2024-09-17T23:48:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 12950K .......... .......... .......... .......... .......... 7% 58.4M 6s 2024-09-17T23:48:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13000K .......... .......... .......... .......... .......... 7% 30.3M 6s 2024-09-17T23:48:09.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13050K .......... .......... .......... .......... .......... 7% 25.1M 6s 2024-09-17T23:48:09.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13100K .......... .......... .......... .......... .......... 7% 24.9M 6s 2024-09-17T23:48:09.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13150K .......... .......... .......... .......... .......... 7% 24.6M 6s 2024-09-17T23:48:09.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13200K .......... .......... .......... .......... .......... 7% 47.1M 6s 2024-09-17T23:48:09.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13250K .......... .......... .......... .......... .......... 7% 26.5M 6s 2024-09-17T23:48:09.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13300K .......... .......... .......... .......... .......... 7% 27.9M 6s 2024-09-17T23:48:09.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13350K .......... .......... .......... .......... .......... 7% 28.7M 6s 2024-09-17T23:48:09.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13400K .......... .......... .......... .......... .......... 7% 47.6M 6s 2024-09-17T23:48:09.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13450K .......... .......... .......... .......... .......... 8% 25.2M 6s 2024-09-17T23:48:09.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13500K .......... .......... .......... .......... .......... 8% 30.0M 6s 2024-09-17T23:48:09.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13550K .......... .......... .......... .......... .......... 8% 24.0M 6s 2024-09-17T23:48:09.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13600K .......... .......... .......... .......... .......... 8% 21.6M 6s 2024-09-17T23:48:09.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13650K .......... .......... .......... .......... .......... 8% 24.7M 6s 2024-09-17T23:48:09.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13700K .......... .......... .......... .......... .......... 8% 47.7M 6s 2024-09-17T23:48:09.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13750K .......... .......... .......... .......... .......... 8% 24.4M 6s 2024-09-17T23:48:09.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13800K .......... .......... .......... .......... .......... 8% 34.1M 6s 2024-09-17T23:48:09.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13850K .......... .......... .......... .......... .......... 8% 25.3M 6s 2024-09-17T23:48:09.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13900K .......... .......... .......... .......... .......... 8% 47.8M 6s 2024-09-17T23:48:09.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 13950K .......... .......... .......... .......... .......... 8% 24.1M 6s 2024-09-17T23:48:09.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14000K .......... .......... .......... .......... .......... 8% 26.0M 6s 2024-09-17T23:48:09.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14050K .......... .......... .......... .......... .......... 8% 33.3M 6s 2024-09-17T23:48:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14100K .......... .......... .......... .......... .......... 8% 21.2M 6s 2024-09-17T23:48:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14150K .......... .......... .......... .......... .......... 8% 57.7M 6s 2024-09-17T23:48:09.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14200K .......... .......... .......... .......... .......... 8% 27.5M 6s 2024-09-17T23:48:09.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14250K .......... .......... .......... .......... .......... 8% 24.1M 6s 2024-09-17T23:48:09.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14300K .......... .......... .......... .......... .......... 8% 34.1M 6s 2024-09-17T23:48:09.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14350K .......... .......... .......... .......... .......... 8% 24.2M 6s 2024-09-17T23:48:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14400K .......... .......... .......... .......... .......... 8% 25.9M 6s 2024-09-17T23:48:09.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14450K .......... .......... .......... .......... .......... 8% 45.2M 6s 2024-09-17T23:48:09.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14500K .......... .......... .......... .......... .......... 8% 27.0M 6s 2024-09-17T23:48:09.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14550K .......... .......... .......... .......... .......... 8% 33.4M 6s 2024-09-17T23:48:09.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14600K .......... .......... .......... .......... .......... 8% 23.9M 6s 2024-09-17T23:48:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14650K .......... .......... .......... .......... .......... 8% 24.1M 6s 2024-09-17T23:48:09.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14700K .......... .......... .......... .......... .......... 8% 76.8M 6s 2024-09-17T23:48:09.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14750K .......... .......... .......... .......... .......... 8% 22.3M 6s 2024-09-17T23:48:09.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14800K .......... .......... .......... .......... .......... 8% 23.1M 6s 2024-09-17T23:48:09.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14850K .......... .......... .......... .......... .......... 8% 32.2M 6s 2024-09-17T23:48:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14900K .......... .......... .......... .......... .......... 8% 25.8M 6s 2024-09-17T23:48:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 14950K .......... .......... .......... .......... .......... 8% 55.8M 6s 2024-09-17T23:48:09.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15000K .......... .......... .......... .......... .......... 8% 25.9M 6s 2024-09-17T23:48:09.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15050K .......... .......... .......... .......... .......... 8% 24.7M 6s 2024-09-17T23:48:09.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15100K .......... .......... .......... .......... .......... 8% 31.6M 6s 2024-09-17T23:48:09.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15150K .......... .......... .......... .......... .......... 9% 23.7M 6s 2024-09-17T23:48:09.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15200K .......... .......... .......... .......... .......... 9% 23.7M 6s 2024-09-17T23:48:09.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15250K .......... .......... .......... .......... .......... 9% 58.8M 6s 2024-09-17T23:48:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15300K .......... .......... .......... .......... .......... 9% 30.7M 6s 2024-09-17T23:48:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15350K .......... .......... .......... .......... .......... 9% 23.9M 6s 2024-09-17T23:48:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15400K .......... .......... .......... .......... .......... 9% 26.8M 6s 2024-09-17T23:48:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15450K .......... .......... .......... .......... .......... 9% 48.0M 6s 2024-09-17T23:48:09.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15500K .......... .......... .......... .......... .......... 9% 30.8M 6s 2024-09-17T23:48:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15550K .......... .......... .......... .......... .......... 9% 20.3M 6s 2024-09-17T23:48:09.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15600K .......... .......... .......... .......... .......... 9% 34.2M 6s 2024-09-17T23:48:09.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15650K .......... .......... .......... .......... .......... 9% 25.8M 6s 2024-09-17T23:48:09.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15700K .......... .......... .......... .......... .......... 9% 49.2M 6s 2024-09-17T23:48:09.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15750K .......... .......... .......... .......... .......... 9% 24.2M 6s 2024-09-17T23:48:09.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15800K .......... .......... .......... .......... .......... 9% 32.5M 6s 2024-09-17T23:48:09.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15850K .......... .......... .......... .......... .......... 9% 26.0M 6s 2024-09-17T23:48:09.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15900K .......... .......... .......... .......... .......... 9% 24.3M 6s 2024-09-17T23:48:09.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 15950K .......... .......... .......... .......... .......... 9% 24.4M 6s 2024-09-17T23:48:09.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16000K .......... .......... .......... .......... .......... 9% 47.2M 6s 2024-09-17T23:48:09.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16050K .......... .......... .......... .......... .......... 9% 25.0M 6s 2024-09-17T23:48:09.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16100K .......... .......... .......... .......... .......... 9% 33.9M 6s 2024-09-17T23:48:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16150K .......... .......... .......... .......... .......... 9% 24.2M 6s 2024-09-17T23:48:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16200K .......... .......... .......... .......... .......... 9% 51.8M 6s 2024-09-17T23:48:09.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16250K .......... .......... .......... .......... .......... 9% 25.3M 6s 2024-09-17T23:48:09.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16300K .......... .......... .......... .......... .......... 9% 34.8M 6s 2024-09-17T23:48:09.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16350K .......... .......... .......... .......... .......... 9% 23.7M 6s 2024-09-17T23:48:09.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16400K .......... .......... .......... .......... .......... 9% 24.6M 6s 2024-09-17T23:48:09.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16450K .......... .......... .......... .......... .......... 9% 25.6M 6s 2024-09-17T23:48:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16500K .......... .......... .......... .......... .......... 9% 49.4M 6s 2024-09-17T23:48:09.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16550K .......... .......... .......... .......... .......... 9% 31.0M 6s 2024-09-17T23:48:09.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16600K .......... .......... .......... .......... .......... 9% 20.8M 6s 2024-09-17T23:48:09.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16650K .......... .......... .......... .......... .......... 9% 31.2M 6s 2024-09-17T23:48:09.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16700K .......... .......... .......... .......... .......... 9% 50.0M 6s 2024-09-17T23:48:09.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16750K .......... .......... .......... .......... .......... 9% 22.4M 6s 2024-09-17T23:48:09.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16800K .......... .......... .......... .......... .......... 9% 27.0M 6s 2024-09-17T23:48:09.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16850K .......... .......... .......... .......... .......... 10% 34.1M 6s 2024-09-17T23:48:09.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16900K .......... .......... .......... .......... .......... 10% 24.8M 6s 2024-09-17T23:48:09.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 16950K .......... .......... .......... .......... .......... 10% 33.8M 6s 2024-09-17T23:48:09.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17000K .......... .......... .......... .......... .......... 10% 33.8M 6s 2024-09-17T23:48:09.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17050K .......... .......... .......... .......... .......... 10% 31.3M 6s 2024-09-17T23:48:09.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17100K .......... .......... .......... .......... .......... 10% 26.1M 6s 2024-09-17T23:48:09.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17150K .......... .......... .......... .......... .......... 10% 23.3M 6s 2024-09-17T23:48:09.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17200K .......... .......... .......... .......... .......... 10% 25.0M 6s 2024-09-17T23:48:09.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17250K .......... .......... .......... .......... .......... 10% 51.8M 6s 2024-09-17T23:48:09.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17300K .......... .......... .......... .......... .......... 10% 25.3M 6s 2024-09-17T23:48:09.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17350K .......... .......... .......... .......... .......... 10% 32.4M 6s 2024-09-17T23:48:09.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17400K .......... .......... .......... .......... .......... 10% 26.1M 6s 2024-09-17T23:48:09.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17450K .......... .......... .......... .......... .......... 10% 22.4M 6s 2024-09-17T23:48:09.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17500K .......... .......... .......... .......... .......... 10% 63.9M 6s 2024-09-17T23:48:09.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17550K .......... .......... .......... .......... .......... 10% 23.6M 6s 2024-09-17T23:48:09.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17600K .......... .......... .......... .......... .......... 10% 24.0M 6s 2024-09-17T23:48:09.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17650K .......... .......... .......... .......... .......... 10% 34.4M 6s 2024-09-17T23:48:09.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17700K .......... .......... .......... .......... .......... 10% 25.2M 6s 2024-09-17T23:48:09.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17750K .......... .......... .......... .......... .......... 10% 45.9M 6s 2024-09-17T23:48:09.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17800K .......... .......... .......... .......... .......... 10% 24.9M 6s 2024-09-17T23:48:09.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17850K .......... .......... .......... .......... .......... 10% 34.5M 6s 2024-09-17T23:48:09.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17900K .......... .......... .......... .......... .......... 10% 24.4M 6s 2024-09-17T23:48:09.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 17950K .......... .......... .......... .......... .......... 10% 16.9M 6s 2024-09-17T23:48:09.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18000K .......... .......... .......... .......... .......... 10% 33.3M 6s 2024-09-17T23:48:09.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18050K .......... .......... .......... .......... .......... 10% 31.9M 6s 2024-09-17T23:48:09.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18100K .......... .......... .......... .......... .......... 10% 27.5M 6s 2024-09-17T23:48:09.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18150K .......... .......... .......... .......... .......... 10% 28.3M 6s 2024-09-17T23:48:09.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18200K .......... .......... .......... .......... .......... 10% 24.7M 6s 2024-09-17T23:48:09.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18250K .......... .......... .......... .......... .......... 10% 49.8M 6s 2024-09-17T23:48:09.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18300K .......... .......... .......... .......... .......... 10% 25.9M 6s 2024-09-17T23:48:09.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18350K .......... .......... .......... .......... .......... 10% 23.8M 6s 2024-09-17T23:48:09.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18400K .......... .......... .......... .......... .......... 10% 34.8M 6s 2024-09-17T23:48:09.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18450K .......... .......... .......... .......... .......... 10% 28.2M 6s 2024-09-17T23:48:09.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18500K .......... .......... .......... .......... .......... 11% 39.3M 6s 2024-09-17T23:48:09.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18550K .......... .......... .......... .......... .......... 11% 24.8M 6s 2024-09-17T23:48:09.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18600K .......... .......... .......... .......... .......... 11% 31.3M 6s 2024-09-17T23:48:09.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18650K .......... .......... .......... .......... .......... 11% 25.5M 6s 2024-09-17T23:48:09.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18700K .......... .......... .......... .......... .......... 11% 24.5M 6s 2024-09-17T23:48:09.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18750K .......... .......... .......... .......... .......... 11% 51.1M 6s 2024-09-17T23:48:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18800K .......... .......... .......... .......... .......... 11% 25.1M 6s 2024-09-17T23:48:09.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18850K .......... .......... .......... .......... .......... 11% 24.0M 6s 2024-09-17T23:48:09.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18900K .......... .......... .......... .......... .......... 11% 34.7M 6s 2024-09-17T23:48:09.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 18950K .......... .......... .......... .......... .......... 11% 24.3M 6s 2024-09-17T23:48:09.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19000K .......... .......... .......... .......... .......... 11% 56.2M 6s 2024-09-17T23:48:09.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19050K .......... .......... .......... .......... .......... 11% 23.9M 6s 2024-09-17T23:48:09.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19100K .......... .......... .......... .......... .......... 11% 29.4M 6s 2024-09-17T23:48:09.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19150K .......... .......... .......... .......... .......... 11% 22.4M 6s 2024-09-17T23:48:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19200K .......... .......... .......... .......... .......... 11% 24.7M 6s 2024-09-17T23:48:09.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19250K .......... .......... .......... .......... .......... 11% 27.9M 6s 2024-09-17T23:48:09.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19300K .......... .......... .......... .......... .......... 11% 44.4M 6s 2024-09-17T23:48:09.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19350K .......... .......... .......... .......... .......... 11% 24.3M 6s 2024-09-17T23:48:09.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19400K .......... .......... .......... .......... .......... 11% 35.5M 6s 2024-09-17T23:48:09.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19450K .......... .......... .......... .......... .......... 11% 23.8M 6s 2024-09-17T23:48:09.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19500K .......... .......... .......... .......... .......... 11% 51.8M 6s 2024-09-17T23:48:09.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19550K .......... .......... .......... .......... .......... 11% 24.4M 6s 2024-09-17T23:48:09.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19600K .......... .......... .......... .......... .......... 11% 24.9M 6s 2024-09-17T23:48:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19650K .......... .......... .......... .......... .......... 11% 35.8M 6s 2024-09-17T23:48:09.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19700K .......... .......... .......... .......... .......... 11% 23.9M 6s 2024-09-17T23:48:09.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19750K .......... .......... .......... .......... .......... 11% 22.7M 6s 2024-09-17T23:48:09.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19800K .......... .......... .......... .......... .......... 11% 63.3M 6s 2024-09-17T23:48:09.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19850K .......... .......... .......... .......... .......... 11% 32.7M 6s 2024-09-17T23:48:09.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19900K .......... .......... .......... .......... .......... 11% 23.1M 6s 2024-09-17T23:48:09.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 19950K .......... .......... .......... .......... .......... 11% 24.8M 6s 2024-09-17T23:48:09.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20000K .......... .......... .......... .......... .......... 11% 25.7M 5s 2024-09-17T23:48:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20050K .......... .......... .......... .......... .......... 11% 47.7M 5s 2024-09-17T23:48:09.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20100K .......... .......... .......... .......... .......... 11% 24.8M 5s 2024-09-17T23:48:09.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20150K .......... .......... .......... .......... .......... 11% 35.3M 5s 2024-09-17T23:48:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20200K .......... .......... .......... .......... .......... 12% 25.3M 5s 2024-09-17T23:48:09.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20250K .......... .......... .......... .......... .......... 12% 63.2M 5s 2024-09-17T23:48:09.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20300K .......... .......... .......... .......... .......... 12% 22.2M 5s 2024-09-17T23:48:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20350K .......... .......... .......... .......... .......... 12% 23.7M 5s 2024-09-17T23:48:09.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20400K .......... .......... .......... .......... .......... 12% 32.8M 5s 2024-09-17T23:48:09.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20450K .......... .......... .......... .......... .......... 12% 24.7M 5s 2024-09-17T23:48:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20500K .......... .......... .......... .......... .......... 12% 24.2M 5s 2024-09-17T23:48:09.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20550K .......... .......... .......... .......... .......... 12% 49.5M 5s 2024-09-17T23:48:09.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20600K .......... .......... .......... .......... .......... 12% 25.1M 5s 2024-09-17T23:48:09.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20650K .......... .......... .......... .......... .......... 12% 33.6M 5s 2024-09-17T23:48:09.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20700K .......... .......... .......... .......... .......... 12% 25.9M 5s 2024-09-17T23:48:09.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20750K .......... .......... .......... .......... .......... 12% 24.2M 5s 2024-09-17T23:48:09.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20800K .......... .......... .......... .......... .......... 12% 47.6M 5s 2024-09-17T23:48:09.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20850K .......... .......... .......... .......... .......... 12% 25.2M 5s 2024-09-17T23:48:09.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20900K .......... .......... .......... .......... .......... 12% 32.7M 5s 2024-09-17T23:48:09.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 20950K .......... .......... .......... .......... .......... 12% 23.6M 5s 2024-09-17T23:48:09.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21000K .......... .......... .......... .......... .......... 12% 26.8M 5s 2024-09-17T23:48:09.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21050K .......... .......... .......... .......... .......... 12% 46.7M 5s 2024-09-17T23:48:09.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21100K .......... .......... .......... .......... .......... 12% 25.7M 5s 2024-09-17T23:48:09.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21150K .......... .......... .......... .......... .......... 12% 23.1M 5s 2024-09-17T23:48:09.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21200K .......... .......... .......... .......... .......... 12% 35.4M 5s 2024-09-17T23:48:09.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21250K .......... .......... .......... .......... .......... 12% 24.7M 5s 2024-09-17T23:48:09.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21300K .......... .......... .......... .......... .......... 12% 50.0M 5s 2024-09-17T23:48:09.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21350K .......... .......... .......... .......... .......... 12% 24.3M 5s 2024-09-17T23:48:09.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21400K .......... .......... .......... .......... .......... 12% 35.4M 5s 2024-09-17T23:48:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21450K .......... .......... .......... .......... .......... 12% 22.9M 5s 2024-09-17T23:48:09.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21500K .......... .......... .......... .......... .......... 12% 27.1M 5s 2024-09-17T23:48:09.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21550K .......... .......... .......... .......... .......... 12% 27.2M 5s 2024-09-17T23:48:09.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21600K .......... .......... .......... .......... .......... 12% 36.9M 5s 2024-09-17T23:48:09.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21650K .......... .......... .......... .......... .......... 12% 25.1M 5s 2024-09-17T23:48:09.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21700K .......... .......... .......... .......... .......... 12% 38.4M 5s 2024-09-17T23:48:09.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21750K .......... .......... .......... .......... .......... 12% 23.3M 5s 2024-09-17T23:48:09.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21800K .......... .......... .......... .......... .......... 12% 51.8M 5s 2024-09-17T23:48:09.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21850K .......... .......... .......... .......... .......... 12% 24.7M 5s 2024-09-17T23:48:09.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21900K .......... .......... .......... .......... .......... 13% 27.7M 5s 2024-09-17T23:48:09.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 21950K .......... .......... .......... .......... .......... 13% 23.7M 5s 2024-09-17T23:48:09.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22000K .......... .......... .......... .......... .......... 13% 30.3M 5s 2024-09-17T23:48:09.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22050K .......... .......... .......... .......... .......... 13% 24.2M 5s 2024-09-17T23:48:09.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22100K .......... .......... .......... .......... .......... 13% 57.3M 5s 2024-09-17T23:48:09.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22150K .......... .......... .......... .......... .......... 13% 22.8M 5s 2024-09-17T23:48:09.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22200K .......... .......... .......... .......... .......... 13% 36.4M 5s 2024-09-17T23:48:09.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22250K .......... .......... .......... .......... .......... 13% 25.2M 5s 2024-09-17T23:48:09.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22300K .......... .......... .......... .......... .......... 13% 49.3M 5s 2024-09-17T23:48:09.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22350K .......... .......... .......... .......... .......... 13% 22.7M 5s 2024-09-17T23:48:09.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22400K .......... .......... .......... .......... .......... 13% 24.7M 5s 2024-09-17T23:48:09.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22450K .......... .......... .......... .......... .......... 13% 16.6M 5s 2024-09-17T23:48:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22500K .......... .......... .......... .......... .......... 13% 600K 6s 2024-09-17T23:48:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22550K .......... .......... .......... .......... .......... 13% 216M 6s 2024-09-17T23:48:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22600K .......... .......... .......... .......... .......... 13% 247M 6s 2024-09-17T23:48:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22650K .......... .......... .......... .......... .......... 13% 247M 6s 2024-09-17T23:48:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22700K .......... .......... .......... .......... .......... 13% 219M 6s 2024-09-17T23:48:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22750K .......... .......... .......... .......... .......... 13% 198M 6s 2024-09-17T23:48:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22800K .......... .......... .......... .......... .......... 13% 239M 6s 2024-09-17T23:48:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22850K .......... .......... .......... .......... .......... 13% 209M 6s 2024-09-17T23:48:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22900K .......... .......... .......... .......... .......... 13% 212M 6s 2024-09-17T23:48:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 22950K .......... .......... .......... .......... .......... 13% 217M 6s 2024-09-17T23:48:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23000K .......... .......... .......... .......... .......... 13% 223M 6s 2024-09-17T23:48:10.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23050K .......... .......... .......... .......... .......... 13% 4.82M 6s 2024-09-17T23:48:10.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23100K .......... .......... .......... .......... .......... 13% 237M 6s 2024-09-17T23:48:10.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23150K .......... .......... .......... .......... .......... 13% 204M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23200K .......... .......... .......... .......... .......... 13% 248M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23250K .......... .......... .......... .......... .......... 13% 227M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23300K .......... .......... .......... .......... .......... 13% 244M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23350K .......... .......... .......... .......... .......... 13% 225M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23400K .......... .......... .......... .......... .......... 13% 214M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23450K .......... .......... .......... .......... .......... 13% 231M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23500K .......... .......... .......... .......... .......... 13% 242M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23550K .......... .......... .......... .......... .......... 14% 200M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23600K .......... .......... .......... .......... .......... 14% 19.7M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23650K .......... .......... .......... .......... .......... 14% 237M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23700K .......... .......... .......... .......... .......... 14% 243M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23750K .......... .......... .......... .......... .......... 14% 191M 6s 2024-09-17T23:48:10.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23800K .......... .......... .......... .......... .......... 14% 193M 6s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23850K .......... .......... .......... .......... .......... 14% 245M 6s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23900K .......... .......... .......... .......... .......... 14% 240M 6s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 23950K .......... .......... .......... .......... .......... 14% 194M 6s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24000K .......... .......... .......... .......... .......... 14% 242M 6s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24050K .......... .......... .......... .......... .......... 14% 215M 6s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24100K .......... .......... .......... .......... .......... 14% 220M 6s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24150K .......... .......... .......... .......... .......... 14% 226M 6s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24200K .......... .......... .......... .......... .......... 14% 242M 6s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24250K .......... .......... .......... .......... .......... 14% 240M 6s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24300K .......... .......... .......... .......... .......... 14% 235M 6s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24350K .......... .......... .......... .......... .......... 14% 209M 5s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24400K .......... .......... .......... .......... .......... 14% 244M 5s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24450K .......... .......... .......... .......... .......... 14% 230M 5s 2024-09-17T23:48:10.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24500K .......... .......... .......... .......... .......... 14% 236M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24550K .......... .......... .......... .......... .......... 14% 219M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24600K .......... .......... .......... .......... .......... 14% 229M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24650K .......... .......... .......... .......... .......... 14% 244M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24700K .......... .......... .......... .......... .......... 14% 170M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24750K .......... .......... .......... .......... .......... 14% 192M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24800K .......... .......... .......... .......... .......... 14% 241M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24850K .......... .......... .......... .......... .......... 14% 239M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24900K .......... .......... .......... .......... .......... 14% 251M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 24950K .......... .......... .......... .......... .......... 14% 214M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25000K .......... .......... .......... .......... .......... 14% 242M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25050K .......... .......... .......... .......... .......... 14% 250M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25100K .......... .......... .......... .......... .......... 14% 240M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25150K .......... .......... .......... .......... .......... 14% 204M 5s 2024-09-17T23:48:10.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25200K .......... .......... .......... .......... .......... 14% 254M 5s 2024-09-17T23:48:10.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25250K .......... .......... .......... .......... .......... 15% 233M 5s 2024-09-17T23:48:10.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25300K .......... .......... .......... .......... .......... 15% 248M 5s 2024-09-17T23:48:10.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25350K .......... .......... .......... .......... .......... 15% 226M 5s 2024-09-17T23:48:10.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25400K .......... .......... .......... .......... .......... 15% 245M 5s 2024-09-17T23:48:10.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25450K .......... .......... .......... .......... .......... 15% 238M 5s 2024-09-17T23:48:10.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25500K .......... .......... .......... .......... .......... 15% 245M 5s 2024-09-17T23:48:10.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25550K .......... .......... .......... .......... .......... 15% 3.85M 5s 2024-09-17T23:48:10.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25600K .......... .......... .......... .......... .......... 15% 101M 5s 2024-09-17T23:48:10.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25650K .......... .......... .......... .......... .......... 15% 118M 5s 2024-09-17T23:48:10.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25700K .......... .......... .......... .......... .......... 15% 119M 5s 2024-09-17T23:48:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25750K .......... .......... .......... .......... .......... 15% 108M 5s 2024-09-17T23:48:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25800K .......... .......... .......... .......... .......... 15% 131M 5s 2024-09-17T23:48:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25850K .......... .......... .......... .......... .......... 15% 127M 5s 2024-09-17T23:48:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25900K .......... .......... .......... .......... .......... 15% 75.9M 5s 2024-09-17T23:48:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 25950K .......... .......... .......... .......... .......... 15% 22.8M 5s 2024-09-17T23:48:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26000K .......... .......... .......... .......... .......... 15% 24.3M 5s 2024-09-17T23:48:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26050K .......... .......... .......... .......... .......... 15% 37.7M 5s 2024-09-17T23:48:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26100K .......... .......... .......... .......... .......... 15% 24.0M 5s 2024-09-17T23:48:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26150K .......... .......... .......... .......... .......... 15% 41.9M 5s 2024-09-17T23:48:10.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26200K .......... .......... .......... .......... .......... 15% 25.9M 5s 2024-09-17T23:48:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26250K .......... .......... .......... .......... .......... 15% 37.3M 5s 2024-09-17T23:48:10.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26300K .......... .......... .......... .......... .......... 15% 24.7M 5s 2024-09-17T23:48:10.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26350K .......... .......... .......... .......... .......... 15% 23.9M 5s 2024-09-17T23:48:10.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26400K .......... .......... .......... .......... .......... 15% 46.9M 5s 2024-09-17T23:48:10.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26450K .......... .......... .......... .......... .......... 15% 25.1M 5s 2024-09-17T23:48:10.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26500K .......... .......... .......... .......... .......... 15% 36.9M 5s 2024-09-17T23:48:10.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26550K .......... .......... .......... .......... .......... 15% 22.9M 5s 2024-09-17T23:48:10.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26600K .......... .......... .......... .......... .......... 15% 24.6M 5s 2024-09-17T23:48:10.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26650K .......... .......... .......... .......... .......... 15% 53.2M 5s 2024-09-17T23:48:10.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26700K .......... .......... .......... .......... .......... 15% 23.2M 5s 2024-09-17T23:48:10.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26750K .......... .......... .......... .......... .......... 15% 23.9M 5s 2024-09-17T23:48:10.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26800K .......... .......... .......... .......... .......... 15% 35.9M 5s 2024-09-17T23:48:10.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26850K .......... .......... .......... .......... .......... 15% 24.6M 5s 2024-09-17T23:48:10.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26900K .......... .......... .......... .......... .......... 15% 52.4M 5s 2024-09-17T23:48:10.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 26950K .......... .......... .......... .......... .......... 16% 23.7M 5s 2024-09-17T23:48:10.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27000K .......... .......... .......... .......... .......... 16% 38.1M 5s 2024-09-17T23:48:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27050K .......... .......... .......... .......... .......... 16% 24.9M 5s 2024-09-17T23:48:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27100K .......... .......... .......... .......... .......... 16% 24.1M 5s 2024-09-17T23:48:10.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27150K .......... .......... .......... .......... .......... 16% 22.8M 5s 2024-09-17T23:48:10.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27200K .......... .......... .......... .......... .......... 16% 59.5M 5s 2024-09-17T23:48:10.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27250K .......... .......... .......... .......... .......... 16% 21.5M 5s 2024-09-17T23:48:10.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27300K .......... .......... .......... .......... .......... 16% 27.5M 5s 2024-09-17T23:48:10.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27350K .......... .......... .......... .......... .......... 16% 25.9M 5s 2024-09-17T23:48:10.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27400K .......... .......... .......... .......... .......... 16% 106M 5s 2024-09-17T23:48:10.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27450K .......... .......... .......... .......... .......... 16% 23.6M 5s 2024-09-17T23:48:10.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27500K .......... .......... .......... .......... .......... 16% 20.8M 5s 2024-09-17T23:48:10.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27550K .......... .......... .......... .......... .......... 16% 37.5M 5s 2024-09-17T23:48:10.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27600K .......... .......... .......... .......... .......... 16% 21.6M 5s 2024-09-17T23:48:10.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27650K .......... .......... .......... .......... .......... 16% 52.2M 5s 2024-09-17T23:48:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27700K .......... .......... .......... .......... .......... 16% 32.3M 5s 2024-09-17T23:48:10.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27750K .......... .......... .......... .......... .......... 16% 21.6M 5s 2024-09-17T23:48:10.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27800K .......... .......... .......... .......... .......... 16% 21.8M 5s 2024-09-17T23:48:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27850K .......... .......... .......... .......... .......... 16% 40.9M 5s 2024-09-17T23:48:10.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27900K .......... .......... .......... .......... .......... 16% 72.4M 5s 2024-09-17T23:48:10.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 27950K .......... .......... .......... .......... .......... 16% 19.6M 5s 2024-09-17T23:48:10.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28000K .......... .......... .......... .......... .......... 16% 24.9M 5s 2024-09-17T23:48:10.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28050K .......... .......... .......... .......... .......... 16% 25.0M 5s 2024-09-17T23:48:10.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28100K .......... .......... .......... .......... .......... 16% 27.1M 5s 2024-09-17T23:48:10.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28150K .......... .......... .......... .......... .......... 16% 102M 5s 2024-09-17T23:48:10.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28200K .......... .......... .......... .......... .......... 16% 27.1M 5s 2024-09-17T23:48:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28250K .......... .......... .......... .......... .......... 16% 24.1M 5s 2024-09-17T23:48:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28300K .......... .......... .......... .......... .......... 16% 26.2M 5s 2024-09-17T23:48:10.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28350K .......... .......... .......... .......... .......... 16% 24.9M 5s 2024-09-17T23:48:10.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28400K .......... .......... .......... .......... .......... 16% 26.2M 5s 2024-09-17T23:48:10.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28450K .......... .......... .......... .......... .......... 16% 94.4M 5s 2024-09-17T23:48:10.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28500K .......... .......... .......... .......... .......... 16% 26.3M 5s 2024-09-17T23:48:10.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28550K .......... .......... .......... .......... .......... 16% 25.4M 5s 2024-09-17T23:48:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28600K .......... .......... .......... .......... .......... 16% 24.8M 5s 2024-09-17T23:48:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28650K .......... .......... .......... .......... .......... 17% 82.2M 5s 2024-09-17T23:48:10.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28700K .......... .......... .......... .......... .......... 17% 22.0M 5s 2024-09-17T23:48:10.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28750K .......... .......... .......... .......... .......... 17% 24.1M 5s 2024-09-17T23:48:10.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28800K .......... .......... .......... .......... .......... 17% 33.5M 5s 2024-09-17T23:48:10.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28850K .......... .......... .......... .......... .......... 17% 25.5M 5s 2024-09-17T23:48:10.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28900K .......... .......... .......... .......... .......... 17% 26.4M 5s 2024-09-17T23:48:10.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 28950K .......... .......... .......... .......... .......... 17% 43.7M 5s 2024-09-17T23:48:10.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29000K .......... .......... .......... .......... .......... 17% 24.2M 5s 2024-09-17T23:48:10.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29050K .......... .......... .......... .......... .......... 17% 36.9M 5s 2024-09-17T23:48:10.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29100K .......... .......... .......... .......... .......... 17% 24.8M 5s 2024-09-17T23:48:10.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29150K .......... .......... .......... .......... .......... 17% 23.9M 5s 2024-09-17T23:48:10.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29200K .......... .......... .......... .......... .......... 17% 30.6M 5s 2024-09-17T23:48:10.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29250K .......... .......... .......... .......... .......... 17% 13.8M 5s 2024-09-17T23:48:10.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29300K .......... .......... .......... .......... .......... 17% 38.0M 5s 2024-09-17T23:48:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29350K .......... .......... .......... .......... .......... 17% 22.5M 5s 2024-09-17T23:48:10.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29400K .......... .......... .......... .......... .......... 17% 25.2M 5s 2024-09-17T23:48:10.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29450K .......... .......... .......... .......... .......... 17% 53.9M 5s 2024-09-17T23:48:10.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29500K .......... .......... .......... .......... .......... 17% 24.5M 5s 2024-09-17T23:48:10.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29550K .......... .......... .......... .......... .......... 17% 23.4M 5s 2024-09-17T23:48:10.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29600K .......... .......... .......... .......... .......... 17% 36.6M 5s 2024-09-17T23:48:10.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29650K .......... .......... .......... .......... .......... 17% 24.4M 5s 2024-09-17T23:48:10.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29700K .......... .......... .......... .......... .......... 17% 50.1M 5s 2024-09-17T23:48:10.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29750K .......... .......... .......... .......... .......... 17% 23.8M 5s 2024-09-17T23:48:10.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29800K .......... .......... .......... .......... .......... 17% 37.4M 5s 2024-09-17T23:48:10.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29850K .......... .......... .......... .......... .......... 17% 23.4M 5s 2024-09-17T23:48:10.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29900K .......... .......... .......... .......... .......... 17% 23.5M 5s 2024-09-17T23:48:10.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 29950K .......... .......... .......... .......... .......... 17% 24.1M 5s 2024-09-17T23:48:10.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30000K .......... .......... .......... .......... .......... 17% 60.8M 5s 2024-09-17T23:48:10.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30050K .......... .......... .......... .......... .......... 17% 26.7M 5s 2024-09-17T23:48:10.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30100K .......... .......... .......... .......... .......... 17% 29.1M 5s 2024-09-17T23:48:10.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30150K .......... .......... .......... .......... .......... 17% 24.5M 5s 2024-09-17T23:48:10.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30200K .......... .......... .......... .......... .......... 17% 49.3M 5s 2024-09-17T23:48:10.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30250K .......... .......... .......... .......... .......... 17% 24.5M 5s 2024-09-17T23:48:10.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30300K .......... .......... .......... .......... .......... 18% 38.2M 5s 2024-09-17T23:48:10.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30350K .......... .......... .......... .......... .......... 18% 23.8M 5s 2024-09-17T23:48:10.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30400K .......... .......... .......... .......... .......... 18% 24.4M 5s 2024-09-17T23:48:10.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30450K .......... .......... .......... .......... .......... 18% 22.8M 5s 2024-09-17T23:48:10.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30500K .......... .......... .......... .......... .......... 18% 58.3M 5s 2024-09-17T23:48:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30550K .......... .......... .......... .......... .......... 18% 31.9M 5s 2024-09-17T23:48:10.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30600K .......... .......... .......... .......... .......... 18% 25.8M 5s 2024-09-17T23:48:10.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30650K .......... .......... .......... .......... .......... 18% 23.9M 5s 2024-09-17T23:48:10.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30700K .......... .......... .......... .......... .......... 18% 49.7M 5s 2024-09-17T23:48:10.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30750K .......... .......... .......... .......... .......... 18% 23.9M 5s 2024-09-17T23:48:10.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30800K .......... .......... .......... .......... .......... 18% 24.4M 5s 2024-09-17T23:48:10.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30850K .......... .......... .......... .......... .......... 18% 38.2M 5s 2024-09-17T23:48:10.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30900K .......... .......... .......... .......... .......... 18% 25.1M 5s 2024-09-17T23:48:10.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 30950K .......... .......... .......... .......... .......... 18% 44.4M 5s 2024-09-17T23:48:10.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31000K .......... .......... .......... .......... .......... 18% 25.4M 5s 2024-09-17T23:48:10.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31050K .......... .......... .......... .......... .......... 18% 35.9M 5s 2024-09-17T23:48:10.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31100K .......... .......... .......... .......... .......... 18% 24.8M 5s 2024-09-17T23:48:10.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31150K .......... .......... .......... .......... .......... 18% 21.3M 5s 2024-09-17T23:48:10.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31200K .......... .......... .......... .......... .......... 18% 44.4M 5s 2024-09-17T23:48:10.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31250K .......... .......... .......... .......... .......... 18% 27.6M 5s 2024-09-17T23:48:10.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31300K .......... .......... .......... .......... .......... 18% 22.2M 5s 2024-09-17T23:48:10.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31350K .......... .......... .......... .......... .......... 18% 33.1M 5s 2024-09-17T23:48:10.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31400K .......... .......... .......... .......... .......... 18% 25.5M 5s 2024-09-17T23:48:10.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31450K .......... .......... .......... .......... .......... 18% 51.4M 5s 2024-09-17T23:48:10.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31500K .......... .......... .......... .......... .......... 18% 25.4M 5s 2024-09-17T23:48:10.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31550K .......... .......... .......... .......... .......... 18% 23.8M 5s 2024-09-17T23:48:10.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31600K .......... .......... .......... .......... .......... 18% 38.1M 5s 2024-09-17T23:48:10.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31650K .......... .......... .......... .......... .......... 18% 23.3M 5s 2024-09-17T23:48:10.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31700K .......... .......... .......... .......... .......... 18% 25.1M 5s 2024-09-17T23:48:10.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31750K .......... .......... .......... .......... .......... 18% 36.2M 5s 2024-09-17T23:48:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31800K .......... .......... .......... .......... .......... 18% 28.6M 5s 2024-09-17T23:48:10.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31850K .......... .......... .......... .......... .......... 18% 37.2M 5s 2024-09-17T23:48:10.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31900K .......... .......... .......... .......... .......... 18% 27.9M 5s 2024-09-17T23:48:10.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 31950K .......... .......... .......... .......... .......... 18% 21.8M 5s 2024-09-17T23:48:10.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32000K .......... .......... .......... .......... .......... 19% 45.0M 5s 2024-09-17T23:48:10.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32050K .......... .......... .......... .......... .......... 19% 31.3M 5s 2024-09-17T23:48:10.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32100K .......... .......... .......... .......... .......... 19% 23.3M 5s 2024-09-17T23:48:10.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32150K .......... .......... .......... .......... .......... 19% 30.4M 5s 2024-09-17T23:48:10.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32200K .......... .......... .......... .......... .......... 19% 23.3M 5s 2024-09-17T23:48:10.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32250K .......... .......... .......... .......... .......... 19% 52.4M 5s 2024-09-17T23:48:10.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32300K .......... .......... .......... .......... .......... 19% 27.3M 5s 2024-09-17T23:48:10.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32350K .......... .......... .......... .......... .......... 19% 25.3M 5s 2024-09-17T23:48:10.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32400K .......... .......... .......... .......... .......... 19% 28.6M 5s 2024-09-17T23:48:10.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32450K .......... .......... .......... .......... .......... 19% 25.4M 5s 2024-09-17T23:48:10.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32500K .......... .......... .......... .......... .......... 19% 45.0M 5s 2024-09-17T23:48:10.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32550K .......... .......... .......... .......... .......... 19% 11.4M 5s 2024-09-17T23:48:10.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32600K .......... .......... .......... .......... .......... 19% 80.5M 5s 2024-09-17T23:48:10.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32650K .......... .......... .......... .......... .......... 19% 80.6M 5s 2024-09-17T23:48:10.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32700K .......... .......... .......... .......... .......... 19% 56.2M 5s 2024-09-17T23:48:10.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32750K .......... .......... .......... .......... .......... 19% 15.6M 5s 2024-09-17T23:48:10.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32800K .......... .......... .......... .......... .......... 19% 52.9M 5s 2024-09-17T23:48:10.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32850K .......... .......... .......... .......... .......... 19% 24.6M 5s 2024-09-17T23:48:10.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32900K .......... .......... .......... .......... .......... 19% 39.7M 5s 2024-09-17T23:48:10.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 32950K .......... .......... .......... .......... .......... 19% 28.6M 5s 2024-09-17T23:48:10.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33000K .......... .......... .......... .......... .......... 19% 33.7M 5s 2024-09-17T23:48:10.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33050K .......... .......... .......... .......... .......... 19% 27.0M 5s 2024-09-17T23:48:10.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33100K .......... .......... .......... .......... .......... 19% 30.6M 5s 2024-09-17T23:48:10.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33150K .......... .......... .......... .......... .......... 19% 19.5M 5s 2024-09-17T23:48:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33200K .......... .......... .......... .......... .......... 19% 14.2M 5s 2024-09-17T23:48:10.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33250K .......... .......... .......... .......... .......... 19% 40.4M 5s 2024-09-17T23:48:10.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33300K .......... .......... .......... .......... .......... 19% 25.6M 5s 2024-09-17T23:48:10.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33350K .......... .......... .......... .......... .......... 19% 33.3M 5s 2024-09-17T23:48:10.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33400K .......... .......... .......... .......... .......... 19% 24.4M 5s 2024-09-17T23:48:10.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33450K .......... .......... .......... .......... .......... 19% 24.6M 5s 2024-09-17T23:48:10.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33500K .......... .......... .......... .......... .......... 19% 47.5M 5s 2024-09-17T23:48:10.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33550K .......... .......... .......... .......... .......... 19% 23.9M 5s 2024-09-17T23:48:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33600K .......... .......... .......... .......... .......... 19% 24.9M 5s 2024-09-17T23:48:10.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33650K .......... .......... .......... .......... .......... 19% 37.1M 5s 2024-09-17T23:48:10.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33700K .......... .......... .......... .......... .......... 20% 25.4M 5s 2024-09-17T23:48:10.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33750K .......... .......... .......... .......... .......... 20% 43.5M 5s 2024-09-17T23:48:10.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33800K .......... .......... .......... .......... .......... 20% 24.3M 5s 2024-09-17T23:48:10.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33850K .......... .......... .......... .......... .......... 20% 35.0M 5s 2024-09-17T23:48:10.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33900K .......... .......... .......... .......... .......... 20% 23.2M 5s 2024-09-17T23:48:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 33950K .......... .......... .......... .......... .......... 20% 24.1M 5s 2024-09-17T23:48:10.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34000K .......... .......... .......... .......... .......... 20% 25.2M 5s 2024-09-17T23:48:10.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34050K .......... .......... .......... .......... .......... 20% 46.4M 5s 2024-09-17T23:48:10.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34100K .......... .......... .......... .......... .......... 20% 25.1M 5s 2024-09-17T23:48:10.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34150K .......... .......... .......... .......... .......... 20% 34.3M 5s 2024-09-17T23:48:10.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34200K .......... .......... .......... .......... .......... 20% 24.6M 5s 2024-09-17T23:48:10.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34250K .......... .......... .......... .......... .......... 20% 50.1M 5s 2024-09-17T23:48:10.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34300K .......... .......... .......... .......... .......... 20% 25.2M 5s 2024-09-17T23:48:10.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34350K .......... .......... .......... .......... .......... 20% 19.4M 5s 2024-09-17T23:48:10.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34400K .......... .......... .......... .......... .......... 20% 62.7M 5s 2024-09-17T23:48:10.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34450K .......... .......... .......... .......... .......... 20% 22.8M 5s 2024-09-17T23:48:10.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34500K .......... .......... .......... .......... .......... 20% 25.1M 5s 2024-09-17T23:48:10.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34550K .......... .......... .......... .......... .......... 20% 44.5M 5s 2024-09-17T23:48:10.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34600K .......... .......... .......... .......... .......... 20% 25.0M 5s 2024-09-17T23:48:10.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34650K .......... .......... .......... .......... .......... 20% 38.8M 5s 2024-09-17T23:48:10.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34700K .......... .......... .......... .......... .......... 20% 23.8M 5s 2024-09-17T23:48:10.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34750K .......... .......... .......... .......... .......... 20% 24.0M 5s 2024-09-17T23:48:10.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34800K .......... .......... .......... .......... .......... 20% 48.1M 5s 2024-09-17T23:48:10.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34850K .......... .......... .......... .......... .......... 20% 25.0M 5s 2024-09-17T23:48:10.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34900K .......... .......... .......... .......... .......... 20% 36.0M 5s 2024-09-17T23:48:10.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 34950K .......... .......... .......... .......... .......... 20% 25.1M 5s 2024-09-17T23:48:10.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35000K .......... .......... .......... .......... .......... 20% 21.7M 5s 2024-09-17T23:48:10.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35050K .......... .......... .......... .......... .......... 20% 47.9M 5s 2024-09-17T23:48:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35100K .......... .......... .......... .......... .......... 20% 23.2M 5s 2024-09-17T23:48:10.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35150K .......... .......... .......... .......... .......... 20% 24.7M 5s 2024-09-17T23:48:10.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35200K .......... .......... .......... .......... .......... 20% 36.0M 5s 2024-09-17T23:48:10.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35250K .......... .......... .......... .......... .......... 20% 25.2M 5s 2024-09-17T23:48:10.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35300K .......... .......... .......... .......... .......... 20% 45.7M 5s 2024-09-17T23:48:10.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35350K .......... .......... .......... .......... .......... 21% 24.9M 5s 2024-09-17T23:48:10.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35400K .......... .......... .......... .......... .......... 21% 38.8M 5s 2024-09-17T23:48:10.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35450K .......... .......... .......... .......... .......... 21% 24.5M 5s 2024-09-17T23:48:10.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35500K .......... .......... .......... .......... .......... 21% 46.2M 5s 2024-09-17T23:48:10.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35550K .......... .......... .......... .......... .......... 21% 15.9M 5s 2024-09-17T23:48:10.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35600K .......... .......... .......... .......... .......... 21% 50.1M 5s 2024-09-17T23:48:10.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35650K .......... .......... .......... .......... .......... 21% 23.8M 5s 2024-09-17T23:48:10.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35700K .......... .......... .......... .......... .......... 21% 38.1M 5s 2024-09-17T23:48:10.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35750K .......... .......... .......... .......... .......... 21% 24.3M 5s 2024-09-17T23:48:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35800K .......... .......... .......... .......... .......... 21% 45.7M 5s 2024-09-17T23:48:10.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35850K .......... .......... .......... .......... .......... 21% 24.8M 5s 2024-09-17T23:48:10.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35900K .......... .......... .......... .......... .......... 21% 43.3M 5s 2024-09-17T23:48:10.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 35950K .......... .......... .......... .......... .......... 21% 22.2M 5s 2024-09-17T23:48:10.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36000K .......... .......... .......... .......... .......... 21% 25.7M 5s 2024-09-17T23:48:10.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36050K .......... .......... .......... .......... .......... 21% 45.0M 5s 2024-09-17T23:48:10.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36100K .......... .......... .......... .......... .......... 21% 24.8M 5s 2024-09-17T23:48:10.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36150K .......... .......... .......... .......... .......... 21% 28.0M 5s 2024-09-17T23:48:10.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36200K .......... .......... .......... .......... .......... 21% 30.0M 5s 2024-09-17T23:48:10.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36250K .......... .......... .......... .......... .......... 21% 23.9M 5s 2024-09-17T23:48:10.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36300K .......... .......... .......... .......... .......... 21% 48.0M 5s 2024-09-17T23:48:10.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36350K .......... .......... .......... .......... .......... 21% 23.6M 5s 2024-09-17T23:48:10.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36400K .......... .......... .......... .......... .......... 21% 25.1M 5s 2024-09-17T23:48:10.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36450K .......... .......... .......... .......... .......... 21% 37.6M 5s 2024-09-17T23:48:10.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36500K .......... .......... .......... .......... .......... 21% 24.5M 5s 2024-09-17T23:48:10.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36550K .......... .......... .......... .......... .......... 21% 40.4M 5s 2024-09-17T23:48:10.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36600K .......... .......... .......... .......... .......... 21% 24.9M 5s 2024-09-17T23:48:10.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36650K .......... .......... .......... .......... .......... 21% 38.2M 5s 2024-09-17T23:48:10.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36700K .......... .......... .......... .......... .......... 21% 23.3M 5s 2024-09-17T23:48:10.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36750K .......... .......... .......... .......... .......... 21% 24.6M 5s 2024-09-17T23:48:10.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36800K .......... .......... .......... .......... .......... 21% 24.6M 5s 2024-09-17T23:48:10.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36850K .......... .......... .......... .......... .......... 21% 45.9M 5s 2024-09-17T23:48:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36900K .......... .......... .......... .......... .......... 21% 24.9M 5s 2024-09-17T23:48:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 36950K .......... .......... .......... .......... .......... 21% 34.2M 5s 2024-09-17T23:48:10.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37000K .......... .......... .......... .......... .......... 21% 23.5M 5s 2024-09-17T23:48:10.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37050K .......... .......... .......... .......... .......... 22% 52.5M 5s 2024-09-17T23:48:10.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37100K .......... .......... .......... .......... .......... 22% 20.2M 5s 2024-09-17T23:48:10.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37150K .......... .......... .......... .......... .......... 22% 19.0M 5s 2024-09-17T23:48:10.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37200K .......... .......... .......... .......... .......... 22% 20.2M 5s 2024-09-17T23:48:10.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37250K .......... .......... .......... .......... .......... 22% 23.2M 5s 2024-09-17T23:48:10.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37300K .......... .......... .......... .......... .......... 22% 23.0M 5s 2024-09-17T23:48:10.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37350K .......... .......... .......... .......... .......... 22% 52.2M 5s 2024-09-17T23:48:10.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37400K .......... .......... .......... .......... .......... 22% 23.1M 5s 2024-09-17T23:48:10.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37450K .......... .......... .......... .......... .......... 22% 37.5M 5s 2024-09-17T23:48:10.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37500K .......... .......... .......... .......... .......... 22% 24.9M 5s 2024-09-17T23:48:10.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37550K .......... .......... .......... .......... .......... 22% 24.1M 5s 2024-09-17T23:48:10.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37600K .......... .......... .......... .......... .......... 22% 48.5M 5s 2024-09-17T23:48:10.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37650K .......... .......... .......... .......... .......... 22% 24.3M 5s 2024-09-17T23:48:10.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37700K .......... .......... .......... .......... .......... 22% 37.7M 5s 2024-09-17T23:48:10.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37750K .......... .......... .......... .......... .......... 22% 24.4M 5s 2024-09-17T23:48:10.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37800K .......... .......... .......... .......... .......... 22% 47.4M 5s 2024-09-17T23:48:10.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37850K .......... .......... .......... .......... .......... 22% 24.7M 5s 2024-09-17T23:48:10.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37900K .......... .......... .......... .......... .......... 22% 34.4M 5s 2024-09-17T23:48:10.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 37950K .......... .......... .......... .......... .......... 22% 18.6M 5s 2024-09-17T23:48:10.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38000K .......... .......... .......... .......... .......... 22% 34.2M 5s 2024-09-17T23:48:10.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38050K .......... .......... .......... .......... .......... 22% 25.5M 5s 2024-09-17T23:48:10.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38100K .......... .......... .......... .......... .......... 22% 44.6M 5s 2024-09-17T23:48:10.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38150K .......... .......... .......... .......... .......... 22% 24.9M 5s 2024-09-17T23:48:10.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38200K .......... .......... .......... .......... .......... 22% 36.4M 5s 2024-09-17T23:48:10.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38250K .......... .......... .......... .......... .......... 22% 26.8M 5s 2024-09-17T23:48:10.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38300K .......... .......... .......... .......... .......... 22% 43.6M 5s 2024-09-17T23:48:10.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38350K .......... .......... .......... .......... .......... 22% 23.7M 5s 2024-09-17T23:48:10.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38400K .......... .......... .......... .......... .......... 22% 24.7M 5s 2024-09-17T23:48:10.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38450K .......... .......... .......... .......... .......... 22% 35.1M 5s 2024-09-17T23:48:10.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38500K .......... .......... .......... .......... .......... 22% 24.3M 5s 2024-09-17T23:48:10.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38550K .......... .......... .......... .......... .......... 22% 25.1M 5s 2024-09-17T23:48:10.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38600K .......... .......... .......... .......... .......... 22% 42.9M 5s 2024-09-17T23:48:10.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38650K .......... .......... .......... .......... .......... 22% 25.2M 5s 2024-09-17T23:48:10.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38700K .......... .......... .......... .......... .......... 22% 39.6M 5s 2024-09-17T23:48:10.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38750K .......... .......... .......... .......... .......... 23% 23.2M 5s 2024-09-17T23:48:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38800K .......... .......... .......... .......... .......... 23% 24.6M 5s 2024-09-17T23:48:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38850K .......... .......... .......... .......... .......... 23% 52.7M 5s 2024-09-17T23:48:10.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38900K .......... .......... .......... .......... .......... 23% 23.8M 5s 2024-09-17T23:48:10.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 38950K .......... .......... .......... .......... .......... 23% 36.9M 5s 2024-09-17T23:48:10.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39000K .......... .......... .......... .......... .......... 23% 21.2M 5s 2024-09-17T23:48:10.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39050K .......... .......... .......... .......... .......... 23% 24.8M 5s 2024-09-17T23:48:10.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39100K .......... .......... .......... .......... .......... 23% 48.1M 5s 2024-09-17T23:48:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39150K .......... .......... .......... .......... .......... 23% 24.4M 5s 2024-09-17T23:48:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39200K .......... .......... .......... .......... .......... 23% 24.2M 5s 2024-09-17T23:48:10.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39250K .......... .......... .......... .......... .......... 23% 36.6M 5s 2024-09-17T23:48:10.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39300K .......... .......... .......... .......... .......... 23% 25.4M 5s 2024-09-17T23:48:10.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39350K .......... .......... .......... .......... .......... 23% 40.2M 5s 2024-09-17T23:48:10.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39400K .......... .......... .......... .......... .......... 23% 25.9M 5s 2024-09-17T23:48:10.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39450K .......... .......... .......... .......... .......... 23% 36.7M 5s 2024-09-17T23:48:10.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39500K .......... .......... .......... .......... .......... 23% 25.5M 5s 2024-09-17T23:48:10.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39550K .......... .......... .......... .......... .......... 23% 22.6M 5s 2024-09-17T23:48:10.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39600K .......... .......... .......... .......... .......... 23% 25.7M 5s 2024-09-17T23:48:10.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39650K .......... .......... .......... .......... .......... 23% 42.5M 5s 2024-09-17T23:48:10.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39700K .......... .......... .......... .......... .......... 23% 26.8M 5s 2024-09-17T23:48:10.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39750K .......... .......... .......... .......... .......... 23% 35.2M 5s 2024-09-17T23:48:10.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39800K .......... .......... .......... .......... .......... 23% 24.4M 5s 2024-09-17T23:48:10.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39850K .......... .......... .......... .......... .......... 23% 43.4M 5s 2024-09-17T23:48:10.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39900K .......... .......... .......... .......... .......... 23% 25.5M 5s 2024-09-17T23:48:10.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 39950K .......... .......... .......... .......... .......... 23% 24.5M 5s 2024-09-17T23:48:10.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40000K .......... .......... .......... .......... .......... 23% 37.5M 5s 2024-09-17T23:48:10.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40050K .......... .......... .......... .......... .......... 23% 24.5M 5s 2024-09-17T23:48:10.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40100K .......... .......... .......... .......... .......... 23% 23.9M 5s 2024-09-17T23:48:10.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40150K .......... .......... .......... .......... .......... 23% 45.9M 5s 2024-09-17T23:48:10.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40200K .......... .......... .......... .......... .......... 23% 26.0M 5s 2024-09-17T23:48:10.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40250K .......... .......... .......... .......... .......... 23% 32.6M 5s 2024-09-17T23:48:10.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40300K .......... .......... .......... .......... .......... 23% 25.5M 5s 2024-09-17T23:48:10.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40350K .......... .......... .......... .......... .......... 23% 24.4M 5s 2024-09-17T23:48:10.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40400K .......... .......... .......... .......... .......... 23% 47.1M 5s 2024-09-17T23:48:10.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40450K .......... .......... .......... .......... .......... 24% 24.0M 5s 2024-09-17T23:48:10.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40500K .......... .......... .......... .......... .......... 24% 36.1M 5s 2024-09-17T23:48:10.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40550K .......... .......... .......... .......... .......... 24% 24.8M 5s 2024-09-17T23:48:10.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40600K .......... .......... .......... .......... .......... 24% 35.7M 5s 2024-09-17T23:48:10.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40650K .......... .......... .......... .......... .......... 24% 28.6M 5s 2024-09-17T23:48:10.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40700K .......... .......... .......... .......... .......... 24% 34.8M 5s 2024-09-17T23:48:10.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40750K .......... .......... .......... .......... .......... 24% 18.7M 5s 2024-09-17T23:48:10.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40800K .......... .......... .......... .......... .......... 24% 33.4M 5s 2024-09-17T23:48:10.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40850K .......... .......... .......... .......... .......... 24% 27.1M 5s 2024-09-17T23:48:10.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40900K .......... .......... .......... .......... .......... 24% 45.2M 5s 2024-09-17T23:48:10.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 40950K .......... .......... .......... .......... .......... 24% 22.6M 5s 2024-09-17T23:48:10.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41000K .......... .......... .......... .......... .......... 24% 38.4M 5s 2024-09-17T23:48:10.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41050K .......... .......... .......... .......... .......... 24% 20.5M 5s 2024-09-17T23:48:10.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41100K .......... .......... .......... .......... .......... 24% 44.8M 5s 2024-09-17T23:48:10.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41150K .......... .......... .......... .......... .......... 24% 21.0M 5s 2024-09-17T23:48:10.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41200K .......... .......... .......... .......... .......... 24% 15.7M 5s 2024-09-17T23:48:10.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41250K .......... .......... .......... .......... .......... 24% 28.3M 5s 2024-09-17T23:48:10.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41300K .......... .......... .......... .......... .......... 24% 22.5M 5s 2024-09-17T23:48:10.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41350K .......... .......... .......... .......... .......... 24% 24.6M 5s 2024-09-17T23:48:10.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41400K .......... .......... .......... .......... .......... 24% 50.3M 5s 2024-09-17T23:48:10.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41450K .......... .......... .......... .......... .......... 24% 28.0M 5s 2024-09-17T23:48:10.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41500K .......... .......... .......... .......... .......... 24% 32.4M 5s 2024-09-17T23:48:10.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41550K .......... .......... .......... .......... .......... 24% 23.3M 5s 2024-09-17T23:48:10.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41600K .......... .......... .......... .......... .......... 24% 41.4M 5s 2024-09-17T23:48:10.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41650K .......... .......... .......... .......... .......... 24% 28.8M 5s 2024-09-17T23:48:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41700K .......... .......... .......... .......... .......... 24% 25.4M 5s 2024-09-17T23:48:10.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41750K .......... .......... .......... .......... .......... 24% 22.3M 5s 2024-09-17T23:48:10.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41800K .......... .......... .......... .......... .......... 24% 32.9M 5s 2024-09-17T23:48:10.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41850K .......... .......... .......... .......... .......... 24% 23.7M 5s 2024-09-17T23:48:10.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41900K .......... .......... .......... .......... .......... 24% 60.7M 5s 2024-09-17T23:48:10.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 41950K .......... .......... .......... .......... .......... 24% 23.8M 5s 2024-09-17T23:48:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42000K .......... .......... .......... .......... .......... 24% 27.8M 5s 2024-09-17T23:48:10.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42050K .......... .......... .......... .......... .......... 24% 30.0M 5s 2024-09-17T23:48:10.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42100K .......... .......... .......... .......... .......... 25% 29.0M 5s 2024-09-17T23:48:10.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42150K .......... .......... .......... .......... .......... 25% 34.7M 5s 2024-09-17T23:48:10.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42200K .......... .......... .......... .......... .......... 25% 31.7M 5s 2024-09-17T23:48:10.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42250K .......... .......... .......... .......... .......... 25% 29.1M 5s 2024-09-17T23:48:10.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42300K .......... .......... .......... .......... .......... 25% 25.1M 5s 2024-09-17T23:48:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42350K .......... .......... .......... .......... .......... 25% 24.1M 5s 2024-09-17T23:48:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42400K .......... .......... .......... .......... .......... 25% 38.2M 4s 2024-09-17T23:48:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42450K .......... .......... .......... .......... .......... 25% 27.9M 4s 2024-09-17T23:48:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42500K .......... .......... .......... .......... .......... 25% 22.7M 4s 2024-09-17T23:48:10.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42550K .......... .......... .......... .......... .......... 25% 36.8M 4s 2024-09-17T23:48:10.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42600K .......... .......... .......... .......... .......... 25% 24.4M 4s 2024-09-17T23:48:10.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42650K .......... .......... .......... .......... .......... 25% 46.1M 4s 2024-09-17T23:48:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42700K .......... .......... .......... .......... .......... 25% 30.7M 4s 2024-09-17T23:48:10.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42750K .......... .......... .......... .......... .......... 25% 20.4M 4s 2024-09-17T23:48:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42800K .......... .......... .......... .......... .......... 25% 24.9M 4s 2024-09-17T23:48:10.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42850K .......... .......... .......... .......... .......... 25% 28.7M 4s 2024-09-17T23:48:10.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42900K .......... .......... .......... .......... .......... 25% 95.3M 4s 2024-09-17T23:48:10.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 42950K .......... .......... .......... .......... .......... 25% 21.9M 4s 2024-09-17T23:48:10.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43000K .......... .......... .......... .......... .......... 25% 25.2M 4s 2024-09-17T23:48:10.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43050K .......... .......... .......... .......... .......... 25% 29.9M 4s 2024-09-17T23:48:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43100K .......... .......... .......... .......... .......... 25% 43.8M 4s 2024-09-17T23:48:10.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43150K .......... .......... .......... .......... .......... 25% 17.2M 4s 2024-09-17T23:48:10.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43200K .......... .......... .......... .......... .......... 25% 44.9M 4s 2024-09-17T23:48:10.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43250K .......... .......... .......... .......... .......... 25% 24.6M 4s 2024-09-17T23:48:10.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43300K .......... .......... .......... .......... .......... 25% 39.2M 4s 2024-09-17T23:48:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43350K .......... .......... .......... .......... .......... 25% 24.3M 4s 2024-09-17T23:48:10.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43400K .......... .......... .......... .......... .......... 25% 45.9M 4s 2024-09-17T23:48:10.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43450K .......... .......... .......... .......... .......... 25% 24.9M 4s 2024-09-17T23:48:10.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43500K .......... .......... .......... .......... .......... 25% 39.0M 4s 2024-09-17T23:48:10.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43550K .......... .......... .......... .......... .......... 25% 22.3M 4s 2024-09-17T23:48:10.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43600K .......... .......... .......... .......... .......... 25% 24.0M 4s 2024-09-17T23:48:10.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43650K .......... .......... .......... .......... .......... 25% 25.4M 4s 2024-09-17T23:48:10.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43700K .......... .......... .......... .......... .......... 25% 45.5M 4s 2024-09-17T23:48:10.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43750K .......... .......... .......... .......... .......... 25% 24.2M 4s 2024-09-17T23:48:10.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43800K .......... .......... .......... .......... .......... 26% 25.2M 4s 2024-09-17T23:48:10.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43850K .......... .......... .......... .......... .......... 26% 37.9M 4s 2024-09-17T23:48:10.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43900K .......... .......... .......... .......... .......... 26% 44.8M 4s 2024-09-17T23:48:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 43950K .......... .......... .......... .......... .......... 26% 26.3M 4s 2024-09-17T23:48:10.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44000K .......... .......... .......... .......... .......... 26% 23.7M 4s 2024-09-17T23:48:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44050K .......... .......... .......... .......... .......... 26% 23.3M 4s 2024-09-17T23:48:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44100K .......... .......... .......... .......... .......... 26% 36.3M 4s 2024-09-17T23:48:10.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44150K .......... .......... .......... .......... .......... 26% 25.4M 4s 2024-09-17T23:48:10.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44200K .......... .......... .......... .......... .......... 26% 41.6M 4s 2024-09-17T23:48:10.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44250K .......... .......... .......... .......... .......... 26% 24.8M 4s 2024-09-17T23:48:10.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44300K .......... .......... .......... .......... .......... 26% 39.6M 4s 2024-09-17T23:48:10.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44350K .......... .......... .......... .......... .......... 26% 23.7M 4s 2024-09-17T23:48:10.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44400K .......... .......... .......... .......... .......... 26% 25.8M 4s 2024-09-17T23:48:10.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44450K .......... .......... .......... .......... .......... 26% 39.1M 4s 2024-09-17T23:48:10.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44500K .......... .......... .......... .......... .......... 26% 25.5M 4s 2024-09-17T23:48:10.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44550K .......... .......... .......... .......... .......... 26% 25.7M 4s 2024-09-17T23:48:10.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44600K .......... .......... .......... .......... .......... 26% 34.9M 4s 2024-09-17T23:48:10.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44650K .......... .......... .......... .......... .......... 26% 24.1M 4s 2024-09-17T23:48:10.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44700K .......... .......... .......... .......... .......... 26% 44.0M 4s 2024-09-17T23:48:10.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44750K .......... .......... .......... .......... .......... 26% 23.0M 4s 2024-09-17T23:48:10.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44800K .......... .......... .......... .......... .......... 26% 30.6M 4s 2024-09-17T23:48:10.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44850K .......... .......... .......... .......... .......... 26% 30.0M 4s 2024-09-17T23:48:10.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44900K .......... .......... .......... .......... .......... 26% 25.5M 4s 2024-09-17T23:48:10.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 44950K .......... .......... .......... .......... .......... 26% 50.5M 4s 2024-09-17T23:48:10.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45000K .......... .......... .......... .......... .......... 26% 20.5M 4s 2024-09-17T23:48:10.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45050K .......... .......... .......... .......... .......... 26% 32.5M 4s 2024-09-17T23:48:10.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45100K .......... .......... .......... .......... .......... 26% 23.0M 4s 2024-09-17T23:48:10.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45150K .......... .......... .......... .......... .......... 26% 18.3M 4s 2024-09-17T23:48:10.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45200K .......... .......... .......... .......... .......... 26% 17.6M 4s 2024-09-17T23:48:10.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45250K .......... .......... .......... .......... .......... 26% 34.8M 4s 2024-09-17T23:48:10.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45300K .......... .......... .......... .......... .......... 26% 23.3M 4s 2024-09-17T23:48:10.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45350K .......... .......... .......... .......... .......... 26% 33.7M 4s 2024-09-17T23:48:10.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45400K .......... .......... .......... .......... .......... 26% 24.8M 4s 2024-09-17T23:48:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45450K .......... .......... .......... .......... .......... 26% 49.0M 4s 2024-09-17T23:48:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45500K .......... .......... .......... .......... .......... 27% 24.2M 4s 2024-09-17T23:48:10.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45550K .......... .......... .......... .......... .......... 27% 23.7M 4s 2024-09-17T23:48:10.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45600K .......... .......... .......... .......... .......... 27% 40.4M 4s 2024-09-17T23:48:10.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45650K .......... .......... .......... .......... .......... 27% 24.8M 4s 2024-09-17T23:48:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45700K .......... .......... .......... .......... .......... 27% 45.0M 4s 2024-09-17T23:48:10.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45750K .......... .......... .......... .......... .......... 27% 23.9M 4s 2024-09-17T23:48:10.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45800K .......... .......... .......... .......... .......... 27% 33.9M 4s 2024-09-17T23:48:10.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45850K .......... .......... .......... .......... .......... 27% 33.6M 4s 2024-09-17T23:48:10.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45900K .......... .......... .......... .......... .......... 27% 19.7M 4s 2024-09-17T23:48:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 45950K .......... .......... .......... .......... .......... 27% 27.0M 4s 2024-09-17T23:48:10.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46000K .......... .......... .......... .......... .......... 27% 39.3M 4s 2024-09-17T23:48:10.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46050K .......... .......... .......... .......... .......... 27% 25.1M 4s 2024-09-17T23:48:10.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46100K .......... .......... .......... .......... .......... 27% 35.7M 4s 2024-09-17T23:48:10.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46150K .......... .......... .......... .......... .......... 27% 24.6M 4s 2024-09-17T23:48:10.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46200K .......... .......... .......... .......... .......... 27% 47.3M 4s 2024-09-17T23:48:10.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46250K .......... .......... .......... .......... .......... 27% 24.4M 4s 2024-09-17T23:48:10.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46300K .......... .......... .......... .......... .......... 27% 30.5M 4s 2024-09-17T23:48:10.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46350K .......... .......... .......... .......... .......... 27% 26.2M 4s 2024-09-17T23:48:10.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46400K .......... .......... .......... .......... .......... 27% 25.4M 4s 2024-09-17T23:48:10.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46450K .......... .......... .......... .......... .......... 27% 24.2M 4s 2024-09-17T23:48:10.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46500K .......... .......... .......... .......... .......... 27% 54.5M 4s 2024-09-17T23:48:10.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46550K .......... .......... .......... .......... .......... 27% 23.3M 4s 2024-09-17T23:48:10.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46600K .......... .......... .......... .......... .......... 27% 37.9M 4s 2024-09-17T23:48:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46650K .......... .......... .......... .......... .......... 27% 24.4M 4s 2024-09-17T23:48:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46700K .......... .......... .......... .......... .......... 27% 49.2M 4s 2024-09-17T23:48:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46750K .......... .......... .......... .......... .......... 27% 23.8M 4s 2024-09-17T23:48:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46800K .......... .......... .......... .......... .......... 27% 25.0M 4s 2024-09-17T23:48:10.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46850K .......... .......... .......... .......... .......... 27% 35.5M 4s 2024-09-17T23:48:10.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46900K .......... .......... .......... .......... .......... 27% 23.3M 4s 2024-09-17T23:48:10.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 46950K .......... .......... .......... .......... .......... 27% 25.9M 4s 2024-09-17T23:48:10.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47000K .......... .......... .......... .......... .......... 27% 49.7M 4s 2024-09-17T23:48:10.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47050K .......... .......... .......... .......... .......... 27% 21.2M 4s 2024-09-17T23:48:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47100K .......... .......... .......... .......... .......... 27% 39.2M 4s 2024-09-17T23:48:10.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47150K .......... .......... .......... .......... .......... 28% 23.4M 4s 2024-09-17T23:48:10.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47200K .......... .......... .......... .......... .......... 28% 25.0M 4s 2024-09-17T23:48:10.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47250K .......... .......... .......... .......... .......... 28% 44.5M 4s 2024-09-17T23:48:10.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47300K .......... .......... .......... .......... .......... 28% 25.6M 4s 2024-09-17T23:48:10.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47350K .......... .......... .......... .......... .......... 28% 35.2M 4s 2024-09-17T23:48:10.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47400K .......... .......... .......... .......... .......... 28% 25.3M 4s 2024-09-17T23:48:10.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47450K .......... .......... .......... .......... .......... 28% 22.8M 4s 2024-09-17T23:48:10.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47500K .......... .......... .......... .......... .......... 28% 117M 4s 2024-09-17T23:48:10.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47550K .......... .......... .......... .......... .......... 28% 18.3M 4s 2024-09-17T23:48:10.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47600K .......... .......... .......... .......... .......... 28% 33.1M 4s 2024-09-17T23:48:10.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47650K .......... .......... .......... .......... .......... 28% 27.5M 4s 2024-09-17T23:48:10.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47700K .......... .......... .......... .......... .......... 28% 43.3M 4s 2024-09-17T23:48:10.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47750K .......... .......... .......... .......... .......... 28% 27.1M 4s 2024-09-17T23:48:10.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47800K .......... .......... .......... .......... .......... 28% 23.1M 4s 2024-09-17T23:48:10.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47850K .......... .......... .......... .......... .......... 28% 37.5M 4s 2024-09-17T23:48:10.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47900K .......... .......... .......... .......... .......... 28% 25.5M 4s 2024-09-17T23:48:10.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 47950K .......... .......... .......... .......... .......... 28% 24.1M 4s 2024-09-17T23:48:10.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48000K .......... .......... .......... .......... .......... 28% 23.4M 4s 2024-09-17T23:48:10.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48050K .......... .......... .......... .......... .......... 28% 53.3M 4s 2024-09-17T23:48:10.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48100K .......... .......... .......... .......... .......... 28% 23.3M 4s 2024-09-17T23:48:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48150K .......... .......... .......... .......... .......... 28% 37.6M 4s 2024-09-17T23:48:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48200K .......... .......... .......... .......... .......... 28% 24.4M 4s 2024-09-17T23:48:10.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48250K .......... .......... .......... .......... .......... 28% 46.2M 4s 2024-09-17T23:48:10.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48300K .......... .......... .......... .......... .......... 28% 24.8M 4s 2024-09-17T23:48:10.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48350K .......... .......... .......... .......... .......... 28% 23.6M 4s 2024-09-17T23:48:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48400K .......... .......... .......... .......... .......... 28% 39.5M 4s 2024-09-17T23:48:10.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48450K .......... .......... .......... .......... .......... 28% 24.5M 4s 2024-09-17T23:48:10.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48500K .......... .......... .......... .......... .......... 28% 42.7M 4s 2024-09-17T23:48:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48550K .......... .......... .......... .......... .......... 28% 24.1M 4s 2024-09-17T23:48:10.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48600K .......... .......... .......... .......... .......... 28% 39.9M 4s 2024-09-17T23:48:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48650K .......... .......... .......... .......... .......... 28% 23.7M 4s 2024-09-17T23:48:10.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48700K .......... .......... .......... .......... .......... 28% 24.7M 4s 2024-09-17T23:48:10.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48750K .......... .......... .......... .......... .......... 28% 24.0M 4s 2024-09-17T23:48:10.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48800K .......... .......... .......... .......... .......... 28% 58.3M 4s 2024-09-17T23:48:10.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48850K .......... .......... .......... .......... .......... 29% 29.9M 4s 2024-09-17T23:48:10.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48900K .......... .......... .......... .......... .......... 29% 27.5M 4s 2024-09-17T23:48:10.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 48950K .......... .......... .......... .......... .......... 29% 21.9M 4s 2024-09-17T23:48:10.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49000K .......... .......... .......... .......... .......... 29% 52.2M 4s 2024-09-17T23:48:10.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49050K .......... .......... .......... .......... .......... 29% 19.6M 4s 2024-09-17T23:48:10.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49100K .......... .......... .......... .......... .......... 29% 35.3M 4s 2024-09-17T23:48:10.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49150K .......... .......... .......... .......... .......... 29% 20.3M 4s 2024-09-17T23:48:10.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49200K .......... .......... .......... .......... .......... 29% 19.9M 4s 2024-09-17T23:48:10.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49250K .......... .......... .......... .......... .......... 29% 17.4M 4s 2024-09-17T23:48:10.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49300K .......... .......... .......... .......... .......... 29% 39.2M 4s 2024-09-17T23:48:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49350K .......... .......... .......... .......... .......... 29% 21.6M 4s 2024-09-17T23:48:10.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49400K .......... .......... .......... .......... .......... 29% 38.4M 4s 2024-09-17T23:48:10.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49450K .......... .......... .......... .......... .......... 29% 24.8M 4s 2024-09-17T23:48:10.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49500K .......... .......... .......... .......... .......... 29% 47.9M 4s 2024-09-17T23:48:10.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49550K .......... .......... .......... .......... .......... 29% 22.8M 4s 2024-09-17T23:48:10.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49600K .......... .......... .......... .......... .......... 29% 25.7M 4s 2024-09-17T23:48:10.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49650K .......... .......... .......... .......... .......... 29% 36.5M 4s 2024-09-17T23:48:10.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49700K .......... .......... .......... .......... .......... 29% 25.4M 4s 2024-09-17T23:48:10.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49750K .......... .......... .......... .......... .......... 29% 23.7M 4s 2024-09-17T23:48:10.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49800K .......... .......... .......... .......... .......... 29% 42.4M 4s 2024-09-17T23:48:10.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49850K .......... .......... .......... .......... .......... 29% 38.5M 4s 2024-09-17T23:48:10.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49900K .......... .......... .......... .......... .......... 29% 24.4M 4s 2024-09-17T23:48:10.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 49950K .......... .......... .......... .......... .......... 29% 23.6M 4s 2024-09-17T23:48:10.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50000K .......... .......... .......... .......... .......... 29% 24.2M 4s 2024-09-17T23:48:10.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50050K .......... .......... .......... .......... .......... 29% 47.0M 4s 2024-09-17T23:48:10.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50100K .......... .......... .......... .......... .......... 29% 24.7M 4s 2024-09-17T23:48:10.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50150K .......... .......... .......... .......... .......... 29% 37.0M 4s 2024-09-17T23:48:10.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50200K .......... .......... .......... .......... .......... 29% 25.7M 4s 2024-09-17T23:48:10.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50250K .......... .......... .......... .......... .......... 29% 44.7M 4s 2024-09-17T23:48:10.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50300K .......... .......... .......... .......... .......... 29% 28.8M 4s 2024-09-17T23:48:10.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50350K .......... .......... .......... .......... .......... 29% 21.2M 4s 2024-09-17T23:48:10.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50400K .......... .......... .......... .......... .......... 29% 33.7M 4s 2024-09-17T23:48:10.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50450K .......... .......... .......... .......... .......... 29% 24.5M 4s 2024-09-17T23:48:10.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50500K .......... .......... .......... .......... .......... 29% 25.5M 4s 2024-09-17T23:48:10.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50550K .......... .......... .......... .......... .......... 30% 44.1M 4s 2024-09-17T23:48:10.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50600K .......... .......... .......... .......... .......... 30% 24.7M 4s 2024-09-17T23:48:10.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50650K .......... .......... .......... .......... .......... 30% 38.1M 4s 2024-09-17T23:48:10.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50700K .......... .......... .......... .......... .......... 30% 24.3M 4s 2024-09-17T23:48:10.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50750K .......... .......... .......... .......... .......... 30% 24.1M 4s 2024-09-17T23:48:10.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50800K .......... .......... .......... .......... .......... 30% 48.7M 4s 2024-09-17T23:48:10.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50850K .......... .......... .......... .......... .......... 30% 24.8M 4s 2024-09-17T23:48:10.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50900K .......... .......... .......... .......... .......... 30% 34.5M 4s 2024-09-17T23:48:10.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 50950K .......... .......... .......... .......... .......... 30% 23.6M 4s 2024-09-17T23:48:10.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51000K .......... .......... .......... .......... .......... 30% 23.9M 4s 2024-09-17T23:48:10.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51050K .......... .......... .......... .......... .......... 30% 45.4M 4s 2024-09-17T23:48:10.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51100K .......... .......... .......... .......... .......... 30% 23.4M 4s 2024-09-17T23:48:10.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51150K .......... .......... .......... .......... .......... 30% 24.8M 4s 2024-09-17T23:48:10.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51200K .......... .......... .......... .......... .......... 30% 25.6M 4s 2024-09-17T23:48:10.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51250K .......... .......... .......... .......... .......... 30% 32.8M 4s 2024-09-17T23:48:10.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51300K .......... .......... .......... .......... .......... 30% 45.1M 4s 2024-09-17T23:48:10.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51350K .......... .......... .......... .......... .......... 30% 24.2M 4s 2024-09-17T23:48:10.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51400K .......... .......... .......... .......... .......... 30% 39.6M 4s 2024-09-17T23:48:10.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51450K .......... .......... .......... .......... .......... 30% 23.4M 4s 2024-09-17T23:48:10.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51500K .......... .......... .......... .......... .......... 30% 23.9M 4s 2024-09-17T23:48:10.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51550K .......... .......... .......... .......... .......... 30% 24.7M 4s 2024-09-17T23:48:10.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51600K .......... .......... .......... .......... .......... 30% 46.3M 4s 2024-09-17T23:48:10.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51650K .......... .......... .......... .......... .......... 30% 25.7M 4s 2024-09-17T23:48:10.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51700K .......... .......... .......... .......... .......... 30% 35.8M 4s 2024-09-17T23:48:10.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51750K .......... .......... .......... .......... .......... 30% 22.3M 4s 2024-09-17T23:48:10.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51800K .......... .......... .......... .......... .......... 30% 51.8M 4s 2024-09-17T23:48:10.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51850K .......... .......... .......... .......... .......... 30% 24.1M 4s 2024-09-17T23:48:10.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51900K .......... .......... .......... .......... .......... 30% 38.8M 4s 2024-09-17T23:48:10.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 51950K .......... .......... .......... .......... .......... 30% 24.1M 4s 2024-09-17T23:48:10.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52000K .......... .......... .......... .......... .......... 30% 23.1M 4s 2024-09-17T23:48:10.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52050K .......... .......... .......... .......... .......... 30% 50.1M 4s 2024-09-17T23:48:10.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52100K .......... .......... .......... .......... .......... 30% 26.0M 4s 2024-09-17T23:48:10.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52150K .......... .......... .......... .......... .......... 30% 22.2M 4s 2024-09-17T23:48:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52200K .......... .......... .......... .......... .......... 30% 39.1M 4s 2024-09-17T23:48:10.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52250K .......... .......... .......... .......... .......... 31% 24.5M 4s 2024-09-17T23:48:10.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52300K .......... .......... .......... .......... .......... 31% 48.1M 4s 2024-09-17T23:48:10.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52350K .......... .......... .......... .......... .......... 31% 23.8M 4s 2024-09-17T23:48:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52400K .......... .......... .......... .......... .......... 31% 24.5M 4s 2024-09-17T23:48:10.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52450K .......... .......... .......... .......... .......... 31% 23.6M 4s 2024-09-17T23:48:10.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52500K .......... .......... .......... .......... .......... 31% 39.7M 4s 2024-09-17T23:48:10.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52550K .......... .......... .......... .......... .......... 31% 22.5M 4s 2024-09-17T23:48:10.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52600K .......... .......... .......... .......... .......... 31% 51.8M 4s 2024-09-17T23:48:10.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52650K .......... .......... .......... .......... .......... 31% 23.7M 4s 2024-09-17T23:48:10.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52700K .......... .......... .......... .......... .......... 31% 37.2M 4s 2024-09-17T23:48:10.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52750K .......... .......... .......... .......... .......... 31% 24.0M 4s 2024-09-17T23:48:10.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52800K .......... .......... .......... .......... .......... 31% 25.8M 4s 2024-09-17T23:48:10.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52850K .......... .......... .......... .......... .......... 31% 42.6M 4s 2024-09-17T23:48:10.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52900K .......... .......... .......... .......... .......... 31% 25.2M 4s 2024-09-17T23:48:10.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 52950K .......... .......... .......... .......... .......... 31% 36.0M 4s 2024-09-17T23:48:10.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53000K .......... .......... .......... .......... .......... 31% 23.5M 4s 2024-09-17T23:48:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53050K .......... .......... .......... .......... .......... 31% 38.7M 4s 2024-09-17T23:48:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53100K .......... .......... .......... .......... .......... 31% 21.4M 4s 2024-09-17T23:48:10.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53150K .......... .......... .......... .......... .......... 31% 21.6M 4s 2024-09-17T23:48:10.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53200K .......... .......... .......... .......... .......... 31% 28.1M 4s 2024-09-17T23:48:10.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53250K .......... .......... .......... .......... .......... 31% 22.1M 4s 2024-09-17T23:48:10.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53300K .......... .......... .......... .......... .......... 31% 22.3M 4s 2024-09-17T23:48:11.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53350K .......... .......... .......... .......... .......... 31% 28.5M 4s 2024-09-17T23:48:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53400K .......... .......... .......... .......... .......... 31% 22.4M 4s 2024-09-17T23:48:11.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53450K .......... .......... .......... .......... .......... 31% 34.3M 4s 2024-09-17T23:48:11.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53500K .......... .......... .......... .......... .......... 31% 26.7M 4s 2024-09-17T23:48:11.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53550K .......... .......... .......... .......... .......... 31% 21.8M 4s 2024-09-17T23:48:11.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53600K .......... .......... .......... .......... .......... 31% 45.3M 4s 2024-09-17T23:48:11.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53650K .......... .......... .......... .......... .......... 31% 24.7M 4s 2024-09-17T23:48:11.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53700K .......... .......... .......... .......... .......... 31% 35.8M 4s 2024-09-17T23:48:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53750K .......... .......... .......... .......... .......... 31% 22.8M 4s 2024-09-17T23:48:11.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53800K .......... .......... .......... .......... .......... 31% 25.3M 4s 2024-09-17T23:48:11.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53850K .......... .......... .......... .......... .......... 31% 45.6M 4s 2024-09-17T23:48:11.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53900K .......... .......... .......... .......... .......... 32% 24.5M 4s 2024-09-17T23:48:11.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 53950K .......... .......... .......... .......... .......... 32% 24.3M 4s 2024-09-17T23:48:11.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54000K .......... .......... .......... .......... .......... 32% 39.0M 4s 2024-09-17T23:48:11.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54050K .......... .......... .......... .......... .......... 32% 25.6M 4s 2024-09-17T23:48:11.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54100K .......... .......... .......... .......... .......... 32% 42.4M 4s 2024-09-17T23:48:11.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54150K .......... .......... .......... .......... .......... 32% 24.3M 4s 2024-09-17T23:48:11.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54200K .......... .......... .......... .......... .......... 32% 38.6M 4s 2024-09-17T23:48:11.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54250K .......... .......... .......... .......... .......... 32% 24.4M 4s 2024-09-17T23:48:11.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54300K .......... .......... .......... .......... .......... 32% 23.7M 4s 2024-09-17T23:48:11.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54350K .......... .......... .......... .......... .......... 32% 24.0M 4s 2024-09-17T23:48:11.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54400K .......... .......... .......... .......... .......... 32% 45.5M 4s 2024-09-17T23:48:11.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54450K .......... .......... .......... .......... .......... 32% 25.0M 4s 2024-09-17T23:48:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54500K .......... .......... .......... .......... .......... 32% 37.6M 4s 2024-09-17T23:48:11.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54550K .......... .......... .......... .......... .......... 32% 24.0M 4s 2024-09-17T23:48:11.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54600K .......... .......... .......... .......... .......... 32% 47.5M 4s 2024-09-17T23:48:11.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54650K .......... .......... .......... .......... .......... 32% 27.4M 4s 2024-09-17T23:48:11.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54700K .......... .......... .......... .......... .......... 32% 33.4M 4s 2024-09-17T23:48:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54750K .......... .......... .......... .......... .......... 32% 24.1M 4s 2024-09-17T23:48:11.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54800K .......... .......... .......... .......... .......... 32% 29.5M 4s 2024-09-17T23:48:11.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54850K .......... .......... .......... .......... .......... 32% 22.1M 4s 2024-09-17T23:48:11.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54900K .......... .......... .......... .......... .......... 32% 36.9M 4s 2024-09-17T23:48:11.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 54950K .......... .......... .......... .......... .......... 32% 24.2M 4s 2024-09-17T23:48:11.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55000K .......... .......... .......... .......... .......... 32% 39.5M 4s 2024-09-17T23:48:11.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55050K .......... .......... .......... .......... .......... 32% 23.3M 4s 2024-09-17T23:48:11.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55100K .......... .......... .......... .......... .......... 32% 46.6M 4s 2024-09-17T23:48:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55150K .......... .......... .......... .......... .......... 32% 23.0M 4s 2024-09-17T23:48:11.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55200K .......... .......... .......... .......... .......... 32% 24.8M 4s 2024-09-17T23:48:11.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55250K .......... .......... .......... .......... .......... 32% 39.2M 4s 2024-09-17T23:48:11.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55300K .......... .......... .......... .......... .......... 32% 25.5M 4s 2024-09-17T23:48:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55350K .......... .......... .......... .......... .......... 32% 40.0M 4s 2024-09-17T23:48:11.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55400K .......... .......... .......... .......... .......... 32% 25.3M 4s 2024-09-17T23:48:11.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55450K .......... .......... .......... .......... .......... 32% 35.6M 4s 2024-09-17T23:48:11.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55500K .......... .......... .......... .......... .......... 32% 23.9M 4s 2024-09-17T23:48:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55550K .......... .......... .......... .......... .......... 32% 24.6M 4s 2024-09-17T23:48:11.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55600K .......... .......... .......... .......... .......... 33% 24.9M 4s 2024-09-17T23:48:11.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55650K .......... .......... .......... .......... .......... 33% 36.9M 4s 2024-09-17T23:48:11.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55700K .......... .......... .......... .......... .......... 33% 29.2M 4s 2024-09-17T23:48:11.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55750K .......... .......... .......... .......... .......... 33% 36.4M 4s 2024-09-17T23:48:11.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55800K .......... .......... .......... .......... .......... 33% 24.4M 4s 2024-09-17T23:48:11.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55850K .......... .......... .......... .......... .......... 33% 46.5M 4s 2024-09-17T23:48:11.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55900K .......... .......... .......... .......... .......... 33% 24.6M 4s 2024-09-17T23:48:11.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 55950K .......... .......... .......... .......... .......... 33% 24.3M 4s 2024-09-17T23:48:11.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56000K .......... .......... .......... .......... .......... 33% 33.6M 4s 2024-09-17T23:48:11.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56050K .......... .......... .......... .......... .......... 33% 29.5M 4s 2024-09-17T23:48:11.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56100K .......... .......... .......... .......... .......... 33% 21.6M 4s 2024-09-17T23:48:11.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56150K .......... .......... .......... .......... .......... 33% 46.4M 4s 2024-09-17T23:48:11.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56200K .......... .......... .......... .......... .......... 33% 24.3M 4s 2024-09-17T23:48:11.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56250K .......... .......... .......... .......... .......... 33% 38.6M 4s 2024-09-17T23:48:11.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56300K .......... .......... .......... .......... .......... 33% 25.3M 4s 2024-09-17T23:48:11.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56350K .......... .......... .......... .......... .......... 33% 23.4M 4s 2024-09-17T23:48:11.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56400K .......... .......... .......... .......... .......... 33% 48.8M 4s 2024-09-17T23:48:11.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56450K .......... .......... .......... .......... .......... 33% 23.8M 4s 2024-09-17T23:48:11.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56500K .......... .......... .......... .......... .......... 33% 23.2M 4s 2024-09-17T23:48:11.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56550K .......... .......... .......... .......... .......... 33% 36.6M 4s 2024-09-17T23:48:11.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56600K .......... .......... .......... .......... .......... 33% 24.0M 4s 2024-09-17T23:48:11.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56650K .......... .......... .......... .......... .......... 33% 45.9M 4s 2024-09-17T23:48:11.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56700K .......... .......... .......... .......... .......... 33% 24.4M 4s 2024-09-17T23:48:11.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56750K .......... .......... .......... .......... .......... 33% 25.3M 4s 2024-09-17T23:48:11.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56800K .......... .......... .......... .......... .......... 33% 37.1M 4s 2024-09-17T23:48:11.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56850K .......... .......... .......... .......... .......... 33% 25.3M 4s 2024-09-17T23:48:11.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56900K .......... .......... .......... .......... .......... 33% 46.0M 4s 2024-09-17T23:48:11.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 56950K .......... .......... .......... .......... .......... 33% 23.2M 4s 2024-09-17T23:48:11.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57000K .......... .......... .......... .......... .......... 33% 35.9M 4s 2024-09-17T23:48:11.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57050K .......... .......... .......... .......... .......... 33% 23.7M 4s 2024-09-17T23:48:11.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57100K .......... .......... .......... .......... .......... 33% 19.5M 4s 2024-09-17T23:48:11.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57150K .......... .......... .......... .......... .......... 33% 23.1M 4s 2024-09-17T23:48:11.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57200K .......... .......... .......... .......... .......... 33% 41.9M 4s 2024-09-17T23:48:11.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57250K .......... .......... .......... .......... .......... 33% 22.1M 4s 2024-09-17T23:48:11.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57300K .......... .......... .......... .......... .......... 34% 35.9M 4s 2024-09-17T23:48:11.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57350K .......... .......... .......... .......... .......... 34% 19.8M 4s 2024-09-17T23:48:11.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57400K .......... .......... .......... .......... .......... 34% 39.2M 4s 2024-09-17T23:48:11.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57450K .......... .......... .......... .......... .......... 34% 17.9M 4s 2024-09-17T23:48:11.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57500K .......... .......... .......... .......... .......... 34% 35.3M 4s 2024-09-17T23:48:11.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57550K .......... .......... .......... .......... .......... 34% 19.3M 4s 2024-09-17T23:48:11.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57600K .......... .......... .......... .......... .......... 34% 46.2M 4s 2024-09-17T23:48:11.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57650K .......... .......... .......... .......... .......... 34% 17.2M 4s 2024-09-17T23:48:11.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57700K .......... .......... .......... .......... .......... 34% 46.0M 4s 2024-09-17T23:48:11.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57750K .......... .......... .......... .......... .......... 34% 36.1M 4s 2024-09-17T23:48:11.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57800K .......... .......... .......... .......... .......... 34% 25.3M 4s 2024-09-17T23:48:11.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57850K .......... .......... .......... .......... .......... 34% 25.0M 4s 2024-09-17T23:48:11.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57900K .......... .......... .......... .......... .......... 34% 47.2M 4s 2024-09-17T23:48:11.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 57950K .......... .......... .......... .......... .......... 34% 23.8M 4s 2024-09-17T23:48:11.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58000K .......... .......... .......... .......... .......... 34% 25.2M 4s 2024-09-17T23:48:11.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58050K .......... .......... .......... .......... .......... 34% 37.0M 4s 2024-09-17T23:48:11.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58100K .......... .......... .......... .......... .......... 34% 25.2M 4s 2024-09-17T23:48:11.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58150K .......... .......... .......... .......... .......... 34% 44.8M 4s 2024-09-17T23:48:11.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58200K .......... .......... .......... .......... .......... 34% 24.6M 4s 2024-09-17T23:48:11.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58250K .......... .......... .......... .......... .......... 34% 35.3M 4s 2024-09-17T23:48:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58300K .......... .......... .......... .......... .......... 34% 24.3M 4s 2024-09-17T23:48:11.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58350K .......... .......... .......... .......... .......... 34% 24.4M 4s 2024-09-17T23:48:11.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58400K .......... .......... .......... .......... .......... 34% 23.9M 4s 2024-09-17T23:48:11.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58450K .......... .......... .......... .......... .......... 34% 48.3M 4s 2024-09-17T23:48:11.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58500K .......... .......... .......... .......... .......... 34% 25.1M 4s 2024-09-17T23:48:11.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58550K .......... .......... .......... .......... .......... 34% 37.8M 4s 2024-09-17T23:48:11.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58600K .......... .......... .......... .......... .......... 34% 24.3M 4s 2024-09-17T23:48:11.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58650K .......... .......... .......... .......... .......... 34% 48.6M 4s 2024-09-17T23:48:11.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58700K .......... .......... .......... .......... .......... 34% 33.5M 4s 2024-09-17T23:48:11.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58750K .......... .......... .......... .......... .......... 34% 18.7M 4s 2024-09-17T23:48:11.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58800K .......... .......... .......... .......... .......... 34% 34.0M 4s 2024-09-17T23:48:11.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58850K .......... .......... .......... .......... .......... 34% 24.9M 4s 2024-09-17T23:48:11.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58900K .......... .......... .......... .......... .......... 34% 25.5M 4s 2024-09-17T23:48:11.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 58950K .......... .......... .......... .......... .......... 35% 49.1M 4s 2024-09-17T23:48:11.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59000K .......... .......... .......... .......... .......... 35% 28.1M 4s 2024-09-17T23:48:11.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59050K .......... .......... .......... .......... .......... 35% 30.1M 4s 2024-09-17T23:48:11.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59100K .......... .......... .......... .......... .......... 35% 23.9M 4s 2024-09-17T23:48:11.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59150K .......... .......... .......... .......... .......... 35% 24.5M 4s 2024-09-17T23:48:11.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59200K .......... .......... .......... .......... .......... 35% 39.3M 4s 2024-09-17T23:48:11.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59250K .......... .......... .......... .......... .......... 35% 23.5M 4s 2024-09-17T23:48:11.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59300K .......... .......... .......... .......... .......... 35% 25.4M 4s 2024-09-17T23:48:11.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59350K .......... .......... .......... .......... .......... 35% 34.1M 4s 2024-09-17T23:48:11.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59400K .......... .......... .......... .......... .......... 35% 24.7M 4s 2024-09-17T23:48:11.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59450K .......... .......... .......... .......... .......... 35% 44.9M 4s 2024-09-17T23:48:11.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59500K .......... .......... .......... .......... .......... 35% 24.4M 4s 2024-09-17T23:48:11.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59550K .......... .......... .......... .......... .......... 35% 24.6M 4s 2024-09-17T23:48:11.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59600K .......... .......... .......... .......... .......... 35% 38.9M 4s 2024-09-17T23:48:11.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59650K .......... .......... .......... .......... .......... 35% 25.5M 4s 2024-09-17T23:48:11.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59700K .......... .......... .......... .......... .......... 35% 44.1M 4s 2024-09-17T23:48:11.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59750K .......... .......... .......... .......... .......... 35% 23.7M 4s 2024-09-17T23:48:11.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59800K .......... .......... .......... .......... .......... 35% 39.2M 4s 2024-09-17T23:48:11.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59850K .......... .......... .......... .......... .......... 35% 24.1M 4s 2024-09-17T23:48:11.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59900K .......... .......... .......... .......... .......... 35% 23.2M 4s 2024-09-17T23:48:11.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 59950K .......... .......... .......... .......... .......... 35% 23.4M 4s 2024-09-17T23:48:11.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60000K .......... .......... .......... .......... .......... 35% 45.7M 4s 2024-09-17T23:48:11.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60050K .......... .......... .......... .......... .......... 35% 24.9M 4s 2024-09-17T23:48:11.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60100K .......... .......... .......... .......... .......... 35% 40.1M 4s 2024-09-17T23:48:11.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60150K .......... .......... .......... .......... .......... 35% 24.2M 4s 2024-09-17T23:48:11.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60200K .......... .......... .......... .......... .......... 35% 44.0M 4s 2024-09-17T23:48:11.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60250K .......... .......... .......... .......... .......... 35% 26.0M 4s 2024-09-17T23:48:11.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60300K .......... .......... .......... .......... .......... 35% 38.2M 4s 2024-09-17T23:48:11.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60350K .......... .......... .......... .......... .......... 35% 23.5M 4s 2024-09-17T23:48:11.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60400K .......... .......... .......... .......... .......... 35% 24.5M 4s 2024-09-17T23:48:11.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60450K .......... .......... .......... .......... .......... 35% 44.3M 4s 2024-09-17T23:48:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60500K .......... .......... .......... .......... .......... 35% 25.8M 4s 2024-09-17T23:48:11.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60550K .......... .......... .......... .......... .......... 35% 20.6M 4s 2024-09-17T23:48:11.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60600K .......... .......... .......... .......... .......... 35% 39.4M 4s 2024-09-17T23:48:11.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60650K .......... .......... .......... .......... .......... 36% 24.5M 4s 2024-09-17T23:48:11.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60700K .......... .......... .......... .......... .......... 36% 44.2M 4s 2024-09-17T23:48:11.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60750K .......... .......... .......... .......... .......... 36% 24.0M 4s 2024-09-17T23:48:11.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60800K .......... .......... .......... .......... .......... 36% 26.8M 4s 2024-09-17T23:48:11.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60850K .......... .......... .......... .......... .......... 36% 34.5M 4s 2024-09-17T23:48:11.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60900K .......... .......... .......... .......... .......... 36% 25.4M 4s 2024-09-17T23:48:11.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 60950K .......... .......... .......... .......... .......... 36% 48.7M 4s 2024-09-17T23:48:11.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61000K .......... .......... .......... .......... .......... 36% 22.8M 4s 2024-09-17T23:48:11.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61050K .......... .......... .......... .......... .......... 36% 36.6M 4s 2024-09-17T23:48:11.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61100K .......... .......... .......... .......... .......... 36% 22.1M 4s 2024-09-17T23:48:11.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61150K .......... .......... .......... .......... .......... 36% 20.9M 4s 2024-09-17T23:48:11.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61200K .......... .......... .......... .......... .......... 36% 23.0M 4s 2024-09-17T23:48:11.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61250K .......... .......... .......... .......... .......... 36% 44.6M 4s 2024-09-17T23:48:11.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61300K .......... .......... .......... .......... .......... 36% 22.1M 4s 2024-09-17T23:48:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61350K .......... .......... .......... .......... .......... 36% 35.5M 4s 2024-09-17T23:48:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61400K .......... .......... .......... .......... .......... 36% 19.8M 4s 2024-09-17T23:48:11.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61450K .......... .......... .......... .......... .......... 36% 39.0M 4s 2024-09-17T23:48:11.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61500K .......... .......... .......... .......... .......... 36% 21.3M 4s 2024-09-17T23:48:11.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61550K .......... .......... .......... .......... .......... 36% 18.2M 4s 2024-09-17T23:48:11.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61600K .......... .......... .......... .......... .......... 36% 10.6M 4s 2024-09-17T23:48:11.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61650K .......... .......... .......... .......... .......... 36% 72.0M 4s 2024-09-17T23:48:11.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61700K .......... .......... .......... .......... .......... 36% 58.5M 4s 2024-09-17T23:48:11.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61750K .......... .......... .......... .......... .......... 36% 98.3M 4s 2024-09-17T23:48:11.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61800K .......... .......... .......... .......... .......... 36% 23.6M 4s 2024-09-17T23:48:11.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61850K .......... .......... .......... .......... .......... 36% 34.3M 4s 2024-09-17T23:48:11.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61900K .......... .......... .......... .......... .......... 36% 24.9M 4s 2024-09-17T23:48:11.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 61950K .......... .......... .......... .......... .......... 36% 24.1M 4s 2024-09-17T23:48:11.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62000K .......... .......... .......... .......... .......... 36% 45.1M 4s 2024-09-17T23:48:11.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62050K .......... .......... .......... .......... .......... 36% 25.2M 4s 2024-09-17T23:48:11.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62100K .......... .......... .......... .......... .......... 36% 36.3M 4s 2024-09-17T23:48:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62150K .......... .......... .......... .......... .......... 36% 25.6M 4s 2024-09-17T23:48:11.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62200K .......... .......... .......... .......... .......... 36% 25.0M 4s 2024-09-17T23:48:11.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62250K .......... .......... .......... .......... .......... 36% 44.9M 4s 2024-09-17T23:48:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62300K .......... .......... .......... .......... .......... 36% 22.9M 4s 2024-09-17T23:48:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62350K .......... .......... .......... .......... .......... 37% 32.1M 4s 2024-09-17T23:48:11.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62400K .......... .......... .......... .......... .......... 37% 26.0M 4s 2024-09-17T23:48:11.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62450K .......... .......... .......... .......... .......... 37% 39.7M 4s 2024-09-17T23:48:11.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62500K .......... .......... .......... .......... .......... 37% 33.1M 4s 2024-09-17T23:48:11.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62550K .......... .......... .......... .......... .......... 37% 21.9M 4s 2024-09-17T23:48:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62600K .......... .......... .......... .......... .......... 37% 38.7M 4s 2024-09-17T23:48:11.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62650K .......... .......... .......... .......... .......... 37% 25.0M 4s 2024-09-17T23:48:11.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62700K .......... .......... .......... .......... .......... 37% 43.4M 4s 2024-09-17T23:48:11.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62750K .......... .......... .......... .......... .......... 37% 21.7M 4s 2024-09-17T23:48:11.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62800K .......... .......... .......... .......... .......... 37% 30.0M 4s 2024-09-17T23:48:11.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62850K .......... .......... .......... .......... .......... 37% 22.4M 4s 2024-09-17T23:48:11.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62900K .......... .......... .......... .......... .......... 37% 40.2M 4s 2024-09-17T23:48:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 62950K .......... .......... .......... .......... .......... 37% 41.3M 4s 2024-09-17T23:48:11.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63000K .......... .......... .......... .......... .......... 37% 27.0M 4s 2024-09-17T23:48:11.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63050K .......... .......... .......... .......... .......... 37% 27.2M 4s 2024-09-17T23:48:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63100K .......... .......... .......... .......... .......... 37% 25.3M 4s 2024-09-17T23:48:11.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63150K .......... .......... .......... .......... .......... 37% 28.4M 4s 2024-09-17T23:48:11.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63200K .......... .......... .......... .......... .......... 37% 30.0M 4s 2024-09-17T23:48:11.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63250K .......... .......... .......... .......... .......... 37% 35.9M 4s 2024-09-17T23:48:11.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63300K .......... .......... .......... .......... .......... 37% 23.1M 4s 2024-09-17T23:48:11.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63350K .......... .......... .......... .......... .......... 37% 34.1M 4s 2024-09-17T23:48:11.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63400K .......... .......... .......... .......... .......... 37% 24.1M 4s 2024-09-17T23:48:11.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63450K .......... .......... .......... .......... .......... 37% 24.6M 4s 2024-09-17T23:48:11.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63500K .......... .......... .......... .......... .......... 37% 47.3M 4s 2024-09-17T23:48:11.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63550K .......... .......... .......... .......... .......... 37% 23.7M 4s 2024-09-17T23:48:11.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63600K .......... .......... .......... .......... .......... 37% 24.8M 4s 2024-09-17T23:48:11.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63650K .......... .......... .......... .......... .......... 37% 40.4M 4s 2024-09-17T23:48:11.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63700K .......... .......... .......... .......... .......... 37% 25.1M 4s 2024-09-17T23:48:11.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63750K .......... .......... .......... .......... .......... 37% 39.4M 4s 2024-09-17T23:48:11.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63800K .......... .......... .......... .......... .......... 37% 25.9M 4s 2024-09-17T23:48:11.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63850K .......... .......... .......... .......... .......... 37% 38.6M 4s 2024-09-17T23:48:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63900K .......... .......... .......... .......... .......... 37% 23.2M 4s 2024-09-17T23:48:11.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 63950K .......... .......... .......... .......... .......... 37% 23.6M 4s 2024-09-17T23:48:11.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64000K .......... .......... .......... .......... .......... 37% 24.2M 4s 2024-09-17T23:48:11.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64050K .......... .......... .......... .......... .......... 38% 45.8M 4s 2024-09-17T23:48:11.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64100K .......... .......... .......... .......... .......... 38% 25.9M 4s 2024-09-17T23:48:11.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64150K .......... .......... .......... .......... .......... 38% 29.9M 4s 2024-09-17T23:48:11.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64200K .......... .......... .......... .......... .......... 38% 33.1M 4s 2024-09-17T23:48:11.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64250K .......... .......... .......... .......... .......... 38% 39.5M 4s 2024-09-17T23:48:11.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64300K .......... .......... .......... .......... .......... 38% 24.0M 4s 2024-09-17T23:48:11.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64350K .......... .......... .......... .......... .......... 38% 24.4M 4s 2024-09-17T23:48:11.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64400K .......... .......... .......... .......... .......... 38% 39.3M 4s 2024-09-17T23:48:11.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64450K .......... .......... .......... .......... .......... 38% 25.7M 4s 2024-09-17T23:48:11.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64500K .......... .......... .......... .......... .......... 38% 23.3M 4s 2024-09-17T23:48:11.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64550K .......... .......... .......... .......... .......... 38% 41.4M 4s 2024-09-17T23:48:11.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64600K .......... .......... .......... .......... .......... 38% 25.3M 4s 2024-09-17T23:48:11.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64650K .......... .......... .......... .......... .......... 38% 32.9M 4s 2024-09-17T23:48:11.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64700K .......... .......... .......... .......... .......... 38% 26.9M 4s 2024-09-17T23:48:11.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64750K .......... .......... .......... .......... .......... 38% 22.4M 4s 2024-09-17T23:48:11.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64800K .......... .......... .......... .......... .......... 38% 45.4M 4s 2024-09-17T23:48:11.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64850K .......... .......... .......... .......... .......... 38% 27.7M 4s 2024-09-17T23:48:11.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64900K .......... .......... .......... .......... .......... 38% 32.3M 4s 2024-09-17T23:48:11.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 64950K .......... .......... .......... .......... .......... 38% 24.3M 4s 2024-09-17T23:48:11.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65000K .......... .......... .......... .......... .......... 38% 23.0M 4s 2024-09-17T23:48:11.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65050K .......... .......... .......... .......... .......... 38% 50.6M 4s 2024-09-17T23:48:11.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65100K .......... .......... .......... .......... .......... 38% 21.3M 4s 2024-09-17T23:48:11.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65150K .......... .......... .......... .......... .......... 38% 24.4M 4s 2024-09-17T23:48:11.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65200K .......... .......... .......... .......... .......... 38% 31.0M 4s 2024-09-17T23:48:11.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65250K .......... .......... .......... .......... .......... 38% 29.7M 4s 2024-09-17T23:48:11.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65300K .......... .......... .......... .......... .......... 38% 32.8M 4s 2024-09-17T23:48:11.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65350K .......... .......... .......... .......... .......... 38% 21.2M 4s 2024-09-17T23:48:11.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65400K .......... .......... .......... .......... .......... 38% 37.1M 4s 2024-09-17T23:48:11.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65450K .......... .......... .......... .......... .......... 38% 22.7M 4s 2024-09-17T23:48:11.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65500K .......... .......... .......... .......... .......... 38% 41.0M 4s 2024-09-17T23:48:11.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65550K .......... .......... .......... .......... .......... 38% 14.2M 4s 2024-09-17T23:48:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65600K .......... .......... .......... .......... .......... 38% 46.5M 4s 2024-09-17T23:48:11.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65650K .......... .......... .......... .......... .......... 38% 27.1M 4s 2024-09-17T23:48:11.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65700K .......... .......... .......... .......... .......... 39% 18.0M 4s 2024-09-17T23:48:11.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65750K .......... .......... .......... .......... .......... 39% 21.4M 4s 2024-09-17T23:48:11.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65800K .......... .......... .......... .......... .......... 39% 54.5M 4s 2024-09-17T23:48:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65850K .......... .......... .......... .......... .......... 39% 23.4M 4s 2024-09-17T23:48:11.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65900K .......... .......... .......... .......... .......... 39% 23.7M 4s 2024-09-17T23:48:11.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 65950K .......... .......... .......... .......... .......... 39% 35.3M 4s 2024-09-17T23:48:11.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66000K .......... .......... .......... .......... .......... 39% 24.8M 4s 2024-09-17T23:48:11.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66050K .......... .......... .......... .......... .......... 39% 46.8M 4s 2024-09-17T23:48:11.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66100K .......... .......... .......... .......... .......... 39% 19.9M 4s 2024-09-17T23:48:11.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66150K .......... .......... .......... .......... .......... 39% 40.7M 4s 2024-09-17T23:48:11.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66200K .......... .......... .......... .......... .......... 39% 28.4M 4s 2024-09-17T23:48:11.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66250K .......... .......... .......... .......... .......... 39% 24.1M 4s 2024-09-17T23:48:11.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66300K .......... .......... .......... .......... .......... 39% 47.6M 4s 2024-09-17T23:48:11.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66350K .......... .......... .......... .......... .......... 39% 24.4M 4s 2024-09-17T23:48:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66400K .......... .......... .......... .......... .......... 39% 24.8M 4s 2024-09-17T23:48:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66450K .......... .......... .......... .......... .......... 39% 44.4M 4s 2024-09-17T23:48:11.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66500K .......... .......... .......... .......... .......... 39% 22.7M 4s 2024-09-17T23:48:11.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66550K .......... .......... .......... .......... .......... 39% 45.8M 4s 2024-09-17T23:48:11.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66600K .......... .......... .......... .......... .......... 39% 23.6M 4s 2024-09-17T23:48:11.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66650K .......... .......... .......... .......... .......... 39% 38.9M 4s 2024-09-17T23:48:11.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66700K .......... .......... .......... .......... .......... 39% 28.8M 4s 2024-09-17T23:48:11.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66750K .......... .......... .......... .......... .......... 39% 20.1M 4s 2024-09-17T23:48:11.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66800K .......... .......... .......... .......... .......... 39% 25.5M 4s 2024-09-17T23:48:11.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66850K .......... .......... .......... .......... .......... 39% 44.5M 4s 2024-09-17T23:48:11.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66900K .......... .......... .......... .......... .......... 39% 25.8M 4s 2024-09-17T23:48:11.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 66950K .......... .......... .......... .......... .......... 39% 37.6M 4s 2024-09-17T23:48:11.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67000K .......... .......... .......... .......... .......... 39% 24.8M 4s 2024-09-17T23:48:11.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67050K .......... .......... .......... .......... .......... 39% 44.5M 4s 2024-09-17T23:48:11.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67100K .......... .......... .......... .......... .......... 39% 25.4M 4s 2024-09-17T23:48:11.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67150K .......... .......... .......... .......... .......... 39% 22.9M 4s 2024-09-17T23:48:11.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67200K .......... .......... .......... .......... .......... 39% 38.5M 4s 2024-09-17T23:48:11.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67250K .......... .......... .......... .......... .......... 39% 23.5M 4s 2024-09-17T23:48:11.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67300K .......... .......... .......... .......... .......... 39% 23.6M 4s 2024-09-17T23:48:11.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67350K .......... .......... .......... .......... .......... 39% 49.3M 4s 2024-09-17T23:48:11.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67400K .......... .......... .......... .......... .......... 40% 22.7M 4s 2024-09-17T23:48:11.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67450K .......... .......... .......... .......... .......... 40% 35.5M 4s 2024-09-17T23:48:11.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67500K .......... .......... .......... .......... .......... 40% 27.0M 4s 2024-09-17T23:48:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67550K .......... .......... .......... .......... .......... 40% 24.3M 4s 2024-09-17T23:48:11.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67600K .......... .......... .......... .......... .......... 40% 38.1M 4s 2024-09-17T23:48:11.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67650K .......... .......... .......... .......... .......... 40% 25.4M 4s 2024-09-17T23:48:11.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67700K .......... .......... .......... .......... .......... 40% 38.0M 4s 2024-09-17T23:48:11.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67750K .......... .......... .......... .......... .......... 40% 23.7M 4s 2024-09-17T23:48:11.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67800K .......... .......... .......... .......... .......... 40% 47.6M 4s 2024-09-17T23:48:11.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67850K .......... .......... .......... .......... .......... 40% 23.7M 4s 2024-09-17T23:48:11.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67900K .......... .......... .......... .......... .......... 40% 37.8M 4s 2024-09-17T23:48:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 67950K .......... .......... .......... .......... .......... 40% 16.9M 4s 2024-09-17T23:48:11.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68000K .......... .......... .......... .......... .......... 40% 41.3M 4s 2024-09-17T23:48:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68050K .......... .......... .......... .......... .......... 40% 29.9M 4s 2024-09-17T23:48:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68100K .......... .......... .......... .......... .......... 40% 36.1M 4s 2024-09-17T23:48:11.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68150K .......... .......... .......... .......... .......... 40% 23.8M 4s 2024-09-17T23:48:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68200K .......... .......... .......... .......... .......... 40% 36.4M 4s 2024-09-17T23:48:11.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68250K .......... .......... .......... .......... .......... 40% 25.4M 4s 2024-09-17T23:48:11.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68300K .......... .......... .......... .......... .......... 40% 41.5M 4s 2024-09-17T23:48:11.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68350K .......... .......... .......... .......... .......... 40% 25.3M 4s 2024-09-17T23:48:11.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68400K .......... .......... .......... .......... .......... 40% 24.7M 4s 2024-09-17T23:48:11.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68450K .......... .......... .......... .......... .......... 40% 31.3M 4s 2024-09-17T23:48:11.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68500K .......... .......... .......... .......... .......... 40% 24.8M 4s 2024-09-17T23:48:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68550K .......... .......... .......... .......... .......... 40% 25.5M 4s 2024-09-17T23:48:11.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68600K .......... .......... .......... .......... .......... 40% 38.8M 4s 2024-09-17T23:48:11.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68650K .......... .......... .......... .......... .......... 40% 22.3M 4s 2024-09-17T23:48:11.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68700K .......... .......... .......... .......... .......... 40% 39.1M 4s 2024-09-17T23:48:11.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68750K .......... .......... .......... .......... .......... 40% 22.7M 4s 2024-09-17T23:48:11.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68800K .......... .......... .......... .......... .......... 40% 26.2M 4s 2024-09-17T23:48:11.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68850K .......... .......... .......... .......... .......... 40% 41.8M 4s 2024-09-17T23:48:11.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68900K .......... .......... .......... .......... .......... 40% 25.1M 4s 2024-09-17T23:48:11.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 68950K .......... .......... .......... .......... .......... 40% 35.7M 4s 2024-09-17T23:48:11.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69000K .......... .......... .......... .......... .......... 40% 23.8M 4s 2024-09-17T23:48:11.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69050K .......... .......... .......... .......... .......... 40% 26.4M 3s 2024-09-17T23:48:11.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69100K .......... .......... .......... .......... .......... 41% 39.5M 3s 2024-09-17T23:48:11.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69150K .......... .......... .......... .......... .......... 41% 24.7M 3s 2024-09-17T23:48:11.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69200K .......... .......... .......... .......... .......... 41% 24.2M 3s 2024-09-17T23:48:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69250K .......... .......... .......... .......... .......... 41% 39.6M 3s 2024-09-17T23:48:11.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69300K .......... .......... .......... .......... .......... 41% 22.9M 3s 2024-09-17T23:48:11.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69350K .......... .......... .......... .......... .......... 41% 36.6M 3s 2024-09-17T23:48:11.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69400K .......... .......... .......... .......... .......... 41% 21.8M 3s 2024-09-17T23:48:11.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69450K .......... .......... .......... .......... .......... 41% 34.5M 3s 2024-09-17T23:48:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69500K .......... .......... .......... .......... .......... 41% 22.0M 3s 2024-09-17T23:48:11.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69550K .......... .......... .......... .......... .......... 41% 21.4M 3s 2024-09-17T23:48:11.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69600K .......... .......... .......... .......... .......... 41% 22.6M 3s 2024-09-17T23:48:11.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69650K .......... .......... .......... .......... .......... 41% 37.6M 3s 2024-09-17T23:48:11.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69700K .......... .......... .......... .......... .......... 41% 20.2M 3s 2024-09-17T23:48:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69750K .......... .......... .......... .......... .......... 41% 38.5M 3s 2024-09-17T23:48:11.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69800K .......... .......... .......... .......... .......... 41% 22.7M 3s 2024-09-17T23:48:11.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69850K .......... .......... .......... .......... .......... 41% 40.0M 3s 2024-09-17T23:48:11.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69900K .......... .......... .......... .......... .......... 41% 17.5M 3s 2024-09-17T23:48:11.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 69950K .......... .......... .......... .......... .......... 41% 22.5M 3s 2024-09-17T23:48:11.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70000K .......... .......... .......... .......... .......... 41% 35.7M 3s 2024-09-17T23:48:11.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70050K .......... .......... .......... .......... .......... 41% 24.7M 3s 2024-09-17T23:48:11.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70100K .......... .......... .......... .......... .......... 41% 21.9M 3s 2024-09-17T23:48:11.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70150K .......... .......... .......... .......... .......... 41% 52.7M 3s 2024-09-17T23:48:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70200K .......... .......... .......... .......... .......... 41% 35.1M 3s 2024-09-17T23:48:11.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70250K .......... .......... .......... .......... .......... 41% 25.1M 3s 2024-09-17T23:48:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70300K .......... .......... .......... .......... .......... 41% 25.0M 3s 2024-09-17T23:48:11.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70350K .......... .......... .......... .......... .......... 41% 23.6M 3s 2024-09-17T23:48:11.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70400K .......... .......... .......... .......... .......... 41% 44.5M 3s 2024-09-17T23:48:11.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70450K .......... .......... .......... .......... .......... 41% 38.9M 3s 2024-09-17T23:48:11.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70500K .......... .......... .......... .......... .......... 41% 24.3M 3s 2024-09-17T23:48:11.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70550K .......... .......... .......... .......... .......... 41% 25.5M 3s 2024-09-17T23:48:11.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70600K .......... .......... .......... .......... .......... 41% 45.1M 3s 2024-09-17T23:48:11.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70650K .......... .......... .......... .......... .......... 41% 27.1M 3s 2024-09-17T23:48:11.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70700K .......... .......... .......... .......... .......... 41% 31.1M 3s 2024-09-17T23:48:11.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70750K .......... .......... .......... .......... .......... 42% 18.0M 3s 2024-09-17T23:48:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70800K .......... .......... .......... .......... .......... 42% 36.4M 3s 2024-09-17T23:48:11.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70850K .......... .......... .......... .......... .......... 42% 25.1M 3s 2024-09-17T23:48:11.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70900K .......... .......... .......... .......... .......... 42% 62.1M 3s 2024-09-17T23:48:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 70950K .......... .......... .......... .......... .......... 42% 21.6M 3s 2024-09-17T23:48:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71000K .......... .......... .......... .......... .......... 42% 39.6M 3s 2024-09-17T23:48:11.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71050K .......... .......... .......... .......... .......... 42% 23.9M 3s 2024-09-17T23:48:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71100K .......... .......... .......... .......... .......... 42% 49.2M 3s 2024-09-17T23:48:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71150K .......... .......... .......... .......... .......... 42% 23.2M 3s 2024-09-17T23:48:11.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71200K .......... .......... .......... .......... .......... 42% 29.1M 3s 2024-09-17T23:48:11.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71250K .......... .......... .......... .......... .......... 42% 30.5M 3s 2024-09-17T23:48:11.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71300K .......... .......... .......... .......... .......... 42% 23.2M 3s 2024-09-17T23:48:11.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71350K .......... .......... .......... .......... .......... 42% 25.5M 3s 2024-09-17T23:48:11.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71400K .......... .......... .......... .......... .......... 42% 46.6M 3s 2024-09-17T23:48:11.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71450K .......... .......... .......... .......... .......... 42% 23.9M 3s 2024-09-17T23:48:11.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71500K .......... .......... .......... .......... .......... 42% 39.1M 3s 2024-09-17T23:48:11.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71550K .......... .......... .......... .......... .......... 42% 22.7M 3s 2024-09-17T23:48:11.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71600K .......... .......... .......... .......... .......... 42% 23.8M 3s 2024-09-17T23:48:11.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71650K .......... .......... .......... .......... .......... 42% 45.1M 3s 2024-09-17T23:48:11.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71700K .......... .......... .......... .......... .......... 42% 25.8M 3s 2024-09-17T23:48:11.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71750K .......... .......... .......... .......... .......... 42% 35.6M 3s 2024-09-17T23:48:11.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71800K .......... .......... .......... .......... .......... 42% 24.8M 3s 2024-09-17T23:48:11.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71850K .......... .......... .......... .......... .......... 42% 23.1M 3s 2024-09-17T23:48:11.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71900K .......... .......... .......... .......... .......... 42% 93.0M 3s 2024-09-17T23:48:11.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 71950K .......... .......... .......... .......... .......... 42% 19.0M 3s 2024-09-17T23:48:11.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72000K .......... .......... .......... .......... .......... 42% 24.9M 3s 2024-09-17T23:48:11.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72050K .......... .......... .......... .......... .......... 42% 40.2M 3s 2024-09-17T23:48:11.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72100K .......... .......... .......... .......... .......... 42% 24.4M 3s 2024-09-17T23:48:11.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72150K .......... .......... .......... .......... .......... 42% 43.1M 3s 2024-09-17T23:48:11.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72200K .......... .......... .......... .......... .......... 42% 24.3M 3s 2024-09-17T23:48:11.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72250K .......... .......... .......... .......... .......... 42% 41.5M 3s 2024-09-17T23:48:11.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72300K .......... .......... .......... .......... .......... 42% 24.1M 3s 2024-09-17T23:48:11.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72350K .......... .......... .......... .......... .......... 42% 23.6M 3s 2024-09-17T23:48:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72400K .......... .......... .......... .......... .......... 42% 23.9M 3s 2024-09-17T23:48:11.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72450K .......... .......... .......... .......... .......... 43% 42.2M 3s 2024-09-17T23:48:11.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72500K .......... .......... .......... .......... .......... 43% 24.8M 3s 2024-09-17T23:48:11.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72550K .......... .......... .......... .......... .......... 43% 37.8M 3s 2024-09-17T23:48:11.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72600K .......... .......... .......... .......... .......... 43% 25.0M 3s 2024-09-17T23:48:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72650K .......... .......... .......... .......... .......... 43% 44.1M 3s 2024-09-17T23:48:11.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72700K .......... .......... .......... .......... .......... 43% 24.3M 3s 2024-09-17T23:48:11.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72750K .......... .......... .......... .......... .......... 43% 22.4M 3s 2024-09-17T23:48:11.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72800K .......... .......... .......... .......... .......... 43% 38.9M 3s 2024-09-17T23:48:11.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72850K .......... .......... .......... .......... .......... 43% 23.1M 3s 2024-09-17T23:48:11.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72900K .......... .......... .......... .......... .......... 43% 43.4M 3s 2024-09-17T23:48:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 72950K .......... .......... .......... .......... .......... 43% 22.5M 3s 2024-09-17T23:48:11.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73000K .......... .......... .......... .......... .......... 43% 25.2M 3s 2024-09-17T23:48:11.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73050K .......... .......... .......... .......... .......... 43% 40.0M 3s 2024-09-17T23:48:11.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73100K .......... .......... .......... .......... .......... 43% 24.3M 3s 2024-09-17T23:48:11.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73150K .......... .......... .......... .......... .......... 43% 24.2M 3s 2024-09-17T23:48:11.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73200K .......... .......... .......... .......... .......... 43% 42.6M 3s 2024-09-17T23:48:11.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73250K .......... .......... .......... .......... .......... 43% 25.0M 3s 2024-09-17T23:48:11.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73300K .......... .......... .......... .......... .......... 43% 37.8M 3s 2024-09-17T23:48:11.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73350K .......... .......... .......... .......... .......... 43% 23.4M 3s 2024-09-17T23:48:11.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73400K .......... .......... .......... .......... .......... 43% 40.6M 3s 2024-09-17T23:48:11.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73450K .......... .......... .......... .......... .......... 43% 21.4M 3s 2024-09-17T23:48:11.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73500K .......... .......... .......... .......... .......... 43% 35.7M 3s 2024-09-17T23:48:11.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73550K .......... .......... .......... .......... .......... 43% 21.2M 3s 2024-09-17T23:48:11.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73600K .......... .......... .......... .......... .......... 43% 23.7M 3s 2024-09-17T23:48:11.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73650K .......... .......... .......... .......... .......... 43% 22.5M 3s 2024-09-17T23:48:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73700K .......... .......... .......... .......... .......... 43% 45.0M 3s 2024-09-17T23:48:11.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73750K .......... .......... .......... .......... .......... 43% 21.9M 3s 2024-09-17T23:48:11.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73800K .......... .......... .......... .......... .......... 43% 32.5M 3s 2024-09-17T23:48:11.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73850K .......... .......... .......... .......... .......... 43% 22.2M 3s 2024-09-17T23:48:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73900K .......... .......... .......... .......... .......... 43% 34.2M 3s 2024-09-17T23:48:11.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 73950K .......... .......... .......... .......... .......... 43% 22.6M 3s 2024-09-17T23:48:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74000K .......... .......... .......... .......... .......... 43% 25.7M 3s 2024-09-17T23:48:11.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74050K .......... .......... .......... .......... .......... 43% 22.8M 3s 2024-09-17T23:48:11.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74100K .......... .......... .......... .......... .......... 43% 24.3M 3s 2024-09-17T23:48:11.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74150K .......... .......... .......... .......... .......... 44% 21.6M 3s 2024-09-17T23:48:11.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74200K .......... .......... .......... .......... .......... 44% 45.1M 3s 2024-09-17T23:48:11.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74250K .......... .......... .......... .......... .......... 44% 23.4M 3s 2024-09-17T23:48:11.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74300K .......... .......... .......... .......... .......... 44% 49.3M 3s 2024-09-17T23:48:11.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74350K .......... .......... .......... .......... .......... 44% 21.3M 3s 2024-09-17T23:48:11.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74400K .......... .......... .......... .......... .......... 44% 24.8M 3s 2024-09-17T23:48:11.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74450K .......... .......... .......... .......... .......... 44% 45.4M 3s 2024-09-17T23:48:11.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74500K .......... .......... .......... .......... .......... 44% 25.0M 3s 2024-09-17T23:48:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74550K .......... .......... .......... .......... .......... 44% 37.0M 3s 2024-09-17T23:48:11.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74600K .......... .......... .......... .......... .......... 44% 23.9M 3s 2024-09-17T23:48:11.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74650K .......... .......... .......... .......... .......... 44% 25.7M 3s 2024-09-17T23:48:11.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74700K .......... .......... .......... .......... .......... 44% 41.5M 3s 2024-09-17T23:48:11.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74750K .......... .......... .......... .......... .......... 44% 25.4M 3s 2024-09-17T23:48:11.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74800K .......... .......... .......... .......... .......... 44% 23.2M 3s 2024-09-17T23:48:11.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74850K .......... .......... .......... .......... .......... 44% 47.4M 3s 2024-09-17T23:48:11.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74900K .......... .......... .......... .......... .......... 44% 21.5M 3s 2024-09-17T23:48:11.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 74950K .......... .......... .......... .......... .......... 44% 46.0M 3s 2024-09-17T23:48:11.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75000K .......... .......... .......... .......... .......... 44% 24.9M 3s 2024-09-17T23:48:11.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75050K .......... .......... .......... .......... .......... 44% 39.2M 3s 2024-09-17T23:48:11.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75100K .......... .......... .......... .......... .......... 44% 23.7M 3s 2024-09-17T23:48:11.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75150K .......... .......... .......... .......... .......... 44% 24.5M 3s 2024-09-17T23:48:11.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75200K .......... .......... .......... .......... .......... 44% 23.5M 3s 2024-09-17T23:48:11.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75250K .......... .......... .......... .......... .......... 44% 51.4M 3s 2024-09-17T23:48:11.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75300K .......... .......... .......... .......... .......... 44% 23.7M 3s 2024-09-17T23:48:11.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75350K .......... .......... .......... .......... .......... 44% 36.3M 3s 2024-09-17T23:48:11.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75400K .......... .......... .......... .......... .......... 44% 24.8M 3s 2024-09-17T23:48:11.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75450K .......... .......... .......... .......... .......... 44% 47.2M 3s 2024-09-17T23:48:11.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75500K .......... .......... .......... .......... .......... 44% 23.4M 3s 2024-09-17T23:48:11.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75550K .......... .......... .......... .......... .......... 44% 24.2M 3s 2024-09-17T23:48:11.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75600K .......... .......... .......... .......... .......... 44% 37.2M 3s 2024-09-17T23:48:11.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75650K .......... .......... .......... .......... .......... 44% 25.1M 3s 2024-09-17T23:48:11.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75700K .......... .......... .......... .......... .......... 44% 45.5M 3s 2024-09-17T23:48:11.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75750K .......... .......... .......... .......... .......... 44% 22.4M 3s 2024-09-17T23:48:11.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75800K .......... .......... .......... .......... .......... 44% 35.0M 3s 2024-09-17T23:48:11.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75850K .......... .......... .......... .......... .......... 45% 24.9M 3s 2024-09-17T23:48:11.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75900K .......... .......... .......... .......... .......... 45% 23.8M 3s 2024-09-17T23:48:11.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 75950K .......... .......... .......... .......... .......... 45% 24.5M 3s 2024-09-17T23:48:11.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76000K .......... .......... .......... .......... .......... 45% 48.8M 3s 2024-09-17T23:48:11.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76050K .......... .......... .......... .......... .......... 45% 23.8M 3s 2024-09-17T23:48:11.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76100K .......... .......... .......... .......... .......... 45% 36.6M 3s 2024-09-17T23:48:11.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76150K .......... .......... .......... .......... .......... 45% 25.3M 3s 2024-09-17T23:48:11.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76200K .......... .......... .......... .......... .......... 45% 47.3M 3s 2024-09-17T23:48:11.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76250K .......... .......... .......... .......... .......... 45% 24.6M 3s 2024-09-17T23:48:11.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76300K .......... .......... .......... .......... .......... 45% 37.9M 3s 2024-09-17T23:48:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76350K .......... .......... .......... .......... .......... 45% 22.6M 3s 2024-09-17T23:48:11.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76400K .......... .......... .......... .......... .......... 45% 25.4M 3s 2024-09-17T23:48:11.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76450K .......... .......... .......... .......... .......... 45% 25.3M 3s 2024-09-17T23:48:11.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76500K .......... .......... .......... .......... .......... 45% 42.3M 3s 2024-09-17T23:48:11.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76550K .......... .......... .......... .......... .......... 45% 24.5M 3s 2024-09-17T23:48:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76600K .......... .......... .......... .......... .......... 45% 39.8M 3s 2024-09-17T23:48:11.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76650K .......... .......... .......... .......... .......... 45% 22.4M 3s 2024-09-17T23:48:11.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76700K .......... .......... .......... .......... .......... 45% 44.5M 3s 2024-09-17T23:48:11.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76750K .......... .......... .......... .......... .......... 45% 24.1M 3s 2024-09-17T23:48:11.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76800K .......... .......... .......... .......... .......... 45% 24.0M 3s 2024-09-17T23:48:11.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76850K .......... .......... .......... .......... .......... 45% 38.6M 3s 2024-09-17T23:48:11.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76900K .......... .......... .......... .......... .......... 45% 21.8M 3s 2024-09-17T23:48:11.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 76950K .......... .......... .......... .......... .......... 45% 21.5M 3s 2024-09-17T23:48:11.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77000K .......... .......... .......... .......... .......... 45% 46.4M 3s 2024-09-17T23:48:11.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77050K .......... .......... .......... .......... .......... 45% 24.0M 3s 2024-09-17T23:48:11.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77100K .......... .......... .......... .......... .......... 45% 42.6M 3s 2024-09-17T23:48:11.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77150K .......... .......... .......... .......... .......... 45% 23.2M 3s 2024-09-17T23:48:11.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77200K .......... .......... .......... .......... .......... 45% 25.1M 3s 2024-09-17T23:48:11.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77250K .......... .......... .......... .......... .......... 45% 43.5M 3s 2024-09-17T23:48:11.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77300K .......... .......... .......... .......... .......... 45% 25.0M 3s 2024-09-17T23:48:11.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77350K .......... .......... .......... .......... .......... 45% 38.3M 3s 2024-09-17T23:48:11.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77400K .......... .......... .......... .......... .......... 45% 25.5M 3s 2024-09-17T23:48:11.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77450K .......... .......... .......... .......... .......... 45% 41.7M 3s 2024-09-17T23:48:11.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77500K .......... .......... .......... .......... .......... 46% 23.9M 3s 2024-09-17T23:48:11.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77550K .......... .......... .......... .......... .......... 46% 20.1M 3s 2024-09-17T23:48:11.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77600K .......... .......... .......... .......... .......... 46% 21.7M 3s 2024-09-17T23:48:11.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77650K .......... .......... .......... .......... .......... 46% 29.2M 3s 2024-09-17T23:48:11.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77700K .......... .......... .......... .......... .......... 46% 30.5M 3s 2024-09-17T23:48:11.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77750K .......... .......... .......... .......... .......... 46% 40.9M 3s 2024-09-17T23:48:11.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77800K .......... .......... .......... .......... .......... 46% 20.8M 3s 2024-09-17T23:48:11.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77850K .......... .......... .......... .......... .......... 46% 39.1M 3s 2024-09-17T23:48:11.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77900K .......... .......... .......... .......... .......... 46% 24.5M 3s 2024-09-17T23:48:11.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 77950K .......... .......... .......... .......... .......... 46% 19.2M 3s 2024-09-17T23:48:11.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78000K .......... .......... .......... .......... .......... 46% 38.9M 3s 2024-09-17T23:48:11.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78050K .......... .......... .......... .......... .......... 46% 22.7M 3s 2024-09-17T23:48:11.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78100K .......... .......... .......... .......... .......... 46% 22.2M 3s 2024-09-17T23:48:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78150K .......... .......... .......... .......... .......... 46% 32.3M 3s 2024-09-17T23:48:11.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78200K .......... .......... .......... .......... .......... 46% 22.2M 3s 2024-09-17T23:48:11.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78250K .......... .......... .......... .......... .......... 46% 46.5M 3s 2024-09-17T23:48:11.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78300K .......... .......... .......... .......... .......... 46% 18.2M 3s 2024-09-17T23:48:11.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78350K .......... .......... .......... .......... .......... 46% 23.1M 3s 2024-09-17T23:48:11.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78400K .......... .......... .......... .......... .......... 46% 34.5M 3s 2024-09-17T23:48:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78450K .......... .......... .......... .......... .......... 46% 24.3M 3s 2024-09-17T23:48:11.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78500K .......... .......... .......... .......... .......... 46% 42.9M 3s 2024-09-17T23:48:11.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78550K .......... .......... .......... .......... .......... 46% 24.8M 3s 2024-09-17T23:48:11.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78600K .......... .......... .......... .......... .......... 46% 38.4M 3s 2024-09-17T23:48:11.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78650K .......... .......... .......... .......... .......... 46% 23.3M 3s 2024-09-17T23:48:11.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78700K .......... .......... .......... .......... .......... 46% 24.1M 3s 2024-09-17T23:48:11.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78750K .......... .......... .......... .......... .......... 46% 23.7M 3s 2024-09-17T23:48:11.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78800K .......... .......... .......... .......... .......... 46% 48.4M 3s 2024-09-17T23:48:11.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78850K .......... .......... .......... .......... .......... 46% 25.3M 3s 2024-09-17T23:48:11.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78900K .......... .......... .......... .......... .......... 46% 38.2M 3s 2024-09-17T23:48:11.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 78950K .......... .......... .......... .......... .......... 46% 23.5M 3s 2024-09-17T23:48:11.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79000K .......... .......... .......... .......... .......... 46% 46.4M 3s 2024-09-17T23:48:11.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79050K .......... .......... .......... .......... .......... 46% 25.2M 3s 2024-09-17T23:48:11.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79100K .......... .......... .......... .......... .......... 46% 25.4M 3s 2024-09-17T23:48:11.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79150K .......... .......... .......... .......... .......... 46% 35.4M 3s 2024-09-17T23:48:11.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79200K .......... .......... .......... .......... .......... 47% 23.9M 3s 2024-09-17T23:48:11.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79250K .......... .......... .......... .......... .......... 47% 24.3M 3s 2024-09-17T23:48:11.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79300K .......... .......... .......... .......... .......... 47% 45.7M 3s 2024-09-17T23:48:11.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79350K .......... .......... .......... .......... .......... 47% 24.2M 3s 2024-09-17T23:48:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79400K .......... .......... .......... .......... .......... 47% 39.3M 3s 2024-09-17T23:48:11.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79450K .......... .......... .......... .......... .......... 47% 24.9M 3s 2024-09-17T23:48:11.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79500K .......... .......... .......... .......... .......... 47% 44.3M 3s 2024-09-17T23:48:11.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79550K .......... .......... .......... .......... .......... 47% 24.0M 3s 2024-09-17T23:48:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79600K .......... .......... .......... .......... .......... 47% 25.1M 3s 2024-09-17T23:48:11.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79650K .......... .......... .......... .......... .......... 47% 37.7M 3s 2024-09-17T23:48:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79700K .......... .......... .......... .......... .......... 47% 23.2M 3s 2024-09-17T23:48:11.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79750K .......... .......... .......... .......... .......... 47% 22.6M 3s 2024-09-17T23:48:11.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79800K .......... .......... .......... .......... .......... 47% 57.0M 3s 2024-09-17T23:48:11.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79850K .......... .......... .......... .......... .......... 47% 23.1M 3s 2024-09-17T23:48:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79900K .......... .......... .......... .......... .......... 47% 35.0M 3s 2024-09-17T23:48:11.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 79950K .......... .......... .......... .......... .......... 47% 25.4M 3s 2024-09-17T23:48:11.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80000K .......... .......... .......... .......... .......... 47% 24.8M 3s 2024-09-17T23:48:11.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80050K .......... .......... .......... .......... .......... 47% 38.2M 3s 2024-09-17T23:48:11.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80100K .......... .......... .......... .......... .......... 47% 27.7M 3s 2024-09-17T23:48:11.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80150K .......... .......... .......... .......... .......... 47% 31.1M 3s 2024-09-17T23:48:11.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80200K .......... .......... .......... .......... .......... 47% 25.5M 3s 2024-09-17T23:48:11.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80250K .......... .......... .......... .......... .......... 47% 44.8M 3s 2024-09-17T23:48:11.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80300K .......... .......... .......... .......... .......... 47% 26.0M 3s 2024-09-17T23:48:11.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80350K .......... .......... .......... .......... .......... 47% 23.9M 3s 2024-09-17T23:48:11.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80400K .......... .......... .......... .......... .......... 47% 22.6M 3s 2024-09-17T23:48:11.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80450K .......... .......... .......... .......... .......... 47% 39.6M 3s 2024-09-17T23:48:11.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80500K .......... .......... .......... .......... .......... 47% 25.7M 3s 2024-09-17T23:48:11.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80550K .......... .......... .......... .......... .......... 47% 39.2M 3s 2024-09-17T23:48:11.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80600K .......... .......... .......... .......... .......... 47% 25.1M 3s 2024-09-17T23:48:11.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80650K .......... .......... .......... .......... .......... 47% 40.5M 3s 2024-09-17T23:48:11.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80700K .......... .......... .......... .......... .......... 47% 24.7M 3s 2024-09-17T23:48:11.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80750K .......... .......... .......... .......... .......... 47% 24.0M 3s 2024-09-17T23:48:11.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80800K .......... .......... .......... .......... .......... 47% 43.9M 3s 2024-09-17T23:48:11.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80850K .......... .......... .......... .......... .......... 47% 30.9M 3s 2024-09-17T23:48:11.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80900K .......... .......... .......... .......... .......... 48% 27.6M 3s 2024-09-17T23:48:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 80950K .......... .......... .......... .......... .......... 48% 24.0M 3s 2024-09-17T23:48:11.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81000K .......... .......... .......... .......... .......... 48% 23.2M 3s 2024-09-17T23:48:11.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81050K .......... .......... .......... .......... .......... 48% 42.1M 3s 2024-09-17T23:48:11.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81100K .......... .......... .......... .......... .......... 48% 23.3M 3s 2024-09-17T23:48:11.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81150K .......... .......... .......... .......... .......... 48% 23.8M 3s 2024-09-17T23:48:11.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81200K .......... .......... .......... .......... .......... 48% 37.8M 3s 2024-09-17T23:48:11.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81250K .......... .......... .......... .......... .......... 48% 25.9M 3s 2024-09-17T23:48:11.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81300K .......... .......... .......... .......... .......... 48% 41.9M 3s 2024-09-17T23:48:11.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81350K .......... .......... .......... .......... .......... 48% 24.3M 3s 2024-09-17T23:48:11.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81400K .......... .......... .......... .......... .......... 48% 41.3M 3s 2024-09-17T23:48:11.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81450K .......... .......... .......... .......... .......... 48% 23.6M 3s 2024-09-17T23:48:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81500K .......... .......... .......... .......... .......... 48% 24.6M 3s 2024-09-17T23:48:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81550K .......... .......... .......... .......... .......... 48% 22.8M 3s 2024-09-17T23:48:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81600K .......... .......... .......... .......... .......... 48% 43.1M 3s 2024-09-17T23:48:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81650K .......... .......... .......... .......... .......... 48% 38.2M 3s 2024-09-17T23:48:11.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81700K .......... .......... .......... .......... .......... 48% 23.2M 3s 2024-09-17T23:48:11.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81750K .......... .......... .......... .......... .......... 48% 20.7M 3s 2024-09-17T23:48:11.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81800K .......... .......... .......... .......... .......... 48% 43.5M 3s 2024-09-17T23:48:11.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81850K .......... .......... .......... .......... .......... 48% 24.4M 3s 2024-09-17T23:48:11.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81900K .......... .......... .......... .......... .......... 48% 32.9M 3s 2024-09-17T23:48:11.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 81950K .......... .......... .......... .......... .......... 48% 20.8M 3s 2024-09-17T23:48:11.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82000K .......... .......... .......... .......... .......... 48% 21.8M 3s 2024-09-17T23:48:11.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82050K .......... .......... .......... .......... .......... 48% 24.8M 3s 2024-09-17T23:48:11.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82100K .......... .......... .......... .......... .......... 48% 44.8M 3s 2024-09-17T23:48:11.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82150K .......... .......... .......... .......... .......... 48% 19.8M 3s 2024-09-17T23:48:11.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82200K .......... .......... .......... .......... .......... 48% 32.6M 3s 2024-09-17T23:48:11.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82250K .......... .......... .......... .......... .......... 48% 22.2M 3s 2024-09-17T23:48:11.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82300K .......... .......... .......... .......... .......... 48% 52.8M 3s 2024-09-17T23:48:12.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82350K .......... .......... .......... .......... .......... 48% 21.2M 3s 2024-09-17T23:48:12.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82400K .......... .......... .......... .......... .......... 48% 30.2M 3s 2024-09-17T23:48:12.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82450K .......... .......... .......... .......... .......... 48% 25.4M 3s 2024-09-17T23:48:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82500K .......... .......... .......... .......... .......... 48% 18.1M 3s 2024-09-17T23:48:12.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82550K .......... .......... .......... .......... .......... 49% 21.0M 3s 2024-09-17T23:48:12.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82600K .......... .......... .......... .......... .......... 49% 52.7M 3s 2024-09-17T23:48:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82650K .......... .......... .......... .......... .......... 49% 23.4M 3s 2024-09-17T23:48:12.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82700K .......... .......... .......... .......... .......... 49% 37.3M 3s 2024-09-17T23:48:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82750K .......... .......... .......... .......... .......... 49% 22.5M 3s 2024-09-17T23:48:12.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82800K .......... .......... .......... .......... .......... 49% 24.2M 3s 2024-09-17T23:48:12.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82850K .......... .......... .......... .......... .......... 49% 48.7M 3s 2024-09-17T23:48:12.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82900K .......... .......... .......... .......... .......... 49% 24.8M 3s 2024-09-17T23:48:12.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 82950K .......... .......... .......... .......... .......... 49% 34.3M 3s 2024-09-17T23:48:12.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83000K .......... .......... .......... .......... .......... 49% 26.5M 3s 2024-09-17T23:48:12.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83050K .......... .......... .......... .......... .......... 49% 43.7M 3s 2024-09-17T23:48:12.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83100K .......... .......... .......... .......... .......... 49% 25.6M 3s 2024-09-17T23:48:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83150K .......... .......... .......... .......... .......... 49% 23.9M 3s 2024-09-17T23:48:12.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83200K .......... .......... .......... .......... .......... 49% 35.2M 3s 2024-09-17T23:48:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83250K .......... .......... .......... .......... .......... 49% 24.3M 3s 2024-09-17T23:48:12.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83300K .......... .......... .......... .......... .......... 49% 25.1M 3s 2024-09-17T23:48:12.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83350K .......... .......... .......... .......... .......... 49% 42.3M 3s 2024-09-17T23:48:12.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83400K .......... .......... .......... .......... .......... 49% 38.7M 3s 2024-09-17T23:48:12.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83450K .......... .......... .......... .......... .......... 49% 20.9M 3s 2024-09-17T23:48:12.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83500K .......... .......... .......... .......... .......... 49% 30.7M 3s 2024-09-17T23:48:12.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83550K .......... .......... .......... .......... .......... 49% 24.2M 3s 2024-09-17T23:48:12.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83600K .......... .......... .......... .......... .......... 49% 43.0M 3s 2024-09-17T23:48:12.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83650K .......... .......... .......... .......... .......... 49% 25.3M 3s 2024-09-17T23:48:12.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83700K .......... .......... .......... .......... .......... 49% 38.2M 3s 2024-09-17T23:48:12.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83750K .......... .......... .......... .......... .......... 49% 23.1M 3s 2024-09-17T23:48:12.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83800K .......... .......... .......... .......... .......... 49% 32.0M 3s 2024-09-17T23:48:12.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83850K .......... .......... .......... .......... .......... 49% 36.8M 3s 2024-09-17T23:48:12.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83900K .......... .......... .......... .......... .......... 49% 21.5M 3s 2024-09-17T23:48:12.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 83950K .......... .......... .......... .......... .......... 49% 23.4M 3s 2024-09-17T23:48:12.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84000K .......... .......... .......... .......... .......... 49% 37.3M 3s 2024-09-17T23:48:12.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84050K .......... .......... .......... .......... .......... 49% 25.2M 3s 2024-09-17T23:48:12.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84100K .......... .......... .......... .......... .......... 49% 40.5M 3s 2024-09-17T23:48:12.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84150K .......... .......... .......... .......... .......... 49% 23.2M 3s 2024-09-17T23:48:12.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84200K .......... .......... .......... .......... .......... 49% 39.5M 3s 2024-09-17T23:48:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84250K .......... .......... .......... .......... .......... 50% 24.7M 3s 2024-09-17T23:48:12.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84300K .......... .......... .......... .......... .......... 50% 22.7M 3s 2024-09-17T23:48:12.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84350K .......... .......... .......... .......... .......... 50% 24.3M 3s 2024-09-17T23:48:12.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84400K .......... .......... .......... .......... .......... 50% 44.5M 3s 2024-09-17T23:48:12.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84450K .......... .......... .......... .......... .......... 50% 25.6M 3s 2024-09-17T23:48:12.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84500K .......... .......... .......... .......... .......... 50% 37.5M 3s 2024-09-17T23:48:12.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84550K .......... .......... .......... .......... .......... 50% 24.7M 3s 2024-09-17T23:48:12.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84600K .......... .......... .......... .......... .......... 50% 30.8M 3s 2024-09-17T23:48:12.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84650K .......... .......... .......... .......... .......... 50% 31.5M 3s 2024-09-17T23:48:12.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84700K .......... .......... .......... .......... .......... 50% 46.4M 3s 2024-09-17T23:48:12.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84750K .......... .......... .......... .......... .......... 50% 23.5M 3s 2024-09-17T23:48:12.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84800K .......... .......... .......... .......... .......... 50% 24.7M 3s 2024-09-17T23:48:12.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84850K .......... .......... .......... .......... .......... 50% 23.1M 3s 2024-09-17T23:48:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84900K .......... .......... .......... .......... .......... 50% 45.8M 3s 2024-09-17T23:48:12.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 84950K .......... .......... .......... .......... .......... 50% 24.3M 3s 2024-09-17T23:48:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85000K .......... .......... .......... .......... .......... 50% 39.2M 3s 2024-09-17T23:48:12.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85050K .......... .......... .......... .......... .......... 50% 24.4M 3s 2024-09-17T23:48:12.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85100K .......... .......... .......... .......... .......... 50% 43.5M 3s 2024-09-17T23:48:12.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85150K .......... .......... .......... .......... .......... 50% 21.1M 3s 2024-09-17T23:48:12.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85200K .......... .......... .......... .......... .......... 50% 26.2M 3s 2024-09-17T23:48:12.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85250K .......... .......... .......... .......... .......... 50% 35.9M 3s 2024-09-17T23:48:12.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85300K .......... .......... .......... .......... .......... 50% 23.7M 3s 2024-09-17T23:48:12.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85350K .......... .......... .......... .......... .......... 50% 43.2M 3s 2024-09-17T23:48:12.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85400K .......... .......... .......... .......... .......... 50% 23.9M 3s 2024-09-17T23:48:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85450K .......... .......... .......... .......... .......... 50% 23.9M 3s 2024-09-17T23:48:12.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85500K .......... .......... .......... .......... .......... 50% 43.2M 3s 2024-09-17T23:48:12.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85550K .......... .......... .......... .......... .......... 50% 23.0M 3s 2024-09-17T23:48:12.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85600K .......... .......... .......... .......... .......... 50% 25.2M 3s 2024-09-17T23:48:12.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85650K .......... .......... .......... .......... .......... 50% 42.7M 3s 2024-09-17T23:48:12.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85700K .......... .......... .......... .......... .......... 50% 25.0M 3s 2024-09-17T23:48:12.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85750K .......... .......... .......... .......... .......... 50% 40.6M 3s 2024-09-17T23:48:12.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85800K .......... .......... .......... .......... .......... 50% 22.6M 3s 2024-09-17T23:48:12.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85850K .......... .......... .......... .......... .......... 50% 45.6M 3s 2024-09-17T23:48:12.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85900K .......... .......... .......... .......... .......... 50% 24.3M 3s 2024-09-17T23:48:12.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 85950K .......... .......... .......... .......... .......... 51% 18.6M 3s 2024-09-17T23:48:12.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86000K .......... .......... .......... .......... .......... 51% 22.8M 3s 2024-09-17T23:48:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86050K .......... .......... .......... .......... .......... 51% 33.3M 3s 2024-09-17T23:48:12.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86100K .......... .......... .......... .......... .......... 51% 20.5M 3s 2024-09-17T23:48:12.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86150K .......... .......... .......... .......... .......... 51% 40.6M 3s 2024-09-17T23:48:12.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86200K .......... .......... .......... .......... .......... 51% 25.3M 3s 2024-09-17T23:48:12.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86250K .......... .......... .......... .......... .......... 51% 39.2M 3s 2024-09-17T23:48:12.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86300K .......... .......... .......... .......... .......... 51% 22.0M 3s 2024-09-17T23:48:12.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86350K .......... .......... .......... .......... .......... 51% 21.2M 3s 2024-09-17T23:48:12.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86400K .......... .......... .......... .......... .......... 51% 38.6M 3s 2024-09-17T23:48:12.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86450K .......... .......... .......... .......... .......... 51% 23.0M 3s 2024-09-17T23:48:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86500K .......... .......... .......... .......... .......... 51% 30.4M 3s 2024-09-17T23:48:12.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86550K .......... .......... .......... .......... .......... 51% 23.0M 3s 2024-09-17T23:48:12.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86600K .......... .......... .......... .......... .......... 51% 23.9M 3s 2024-09-17T23:48:12.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86650K .......... .......... .......... .......... .......... 51% 43.8M 3s 2024-09-17T23:48:12.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86700K .......... .......... .......... .......... .......... 51% 23.7M 3s 2024-09-17T23:48:12.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86750K .......... .......... .......... .......... .......... 51% 16.9M 3s 2024-09-17T23:48:12.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86800K .......... .......... .......... .......... .......... 51% 35.2M 3s 2024-09-17T23:48:12.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86850K .......... .......... .......... .......... .......... 51% 25.0M 3s 2024-09-17T23:48:12.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86900K .......... .......... .......... .......... .......... 51% 41.9M 3s 2024-09-17T23:48:12.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 86950K .......... .......... .......... .......... .......... 51% 24.2M 3s 2024-09-17T23:48:12.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87000K .......... .......... .......... .......... .......... 51% 33.4M 3s 2024-09-17T23:48:12.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87050K .......... .......... .......... .......... .......... 51% 21.8M 3s 2024-09-17T23:48:12.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87100K .......... .......... .......... .......... .......... 51% 29.1M 3s 2024-09-17T23:48:12.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87150K .......... .......... .......... .......... .......... 51% 39.5M 3s 2024-09-17T23:48:12.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87200K .......... .......... .......... .......... .......... 51% 27.4M 3s 2024-09-17T23:48:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87250K .......... .......... .......... .......... .......... 51% 22.9M 3s 2024-09-17T23:48:12.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87300K .......... .......... .......... .......... .......... 51% 18.5M 3s 2024-09-17T23:48:12.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87350K .......... .......... .......... .......... .......... 51% 97.4M 3s 2024-09-17T23:48:12.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87400K .......... .......... .......... .......... .......... 51% 40.2M 3s 2024-09-17T23:48:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87450K .......... .......... .......... .......... .......... 51% 24.6M 3s 2024-09-17T23:48:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87500K .......... .......... .......... .......... .......... 51% 26.2M 3s 2024-09-17T23:48:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87550K .......... .......... .......... .......... .......... 51% 35.6M 3s 2024-09-17T23:48:12.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87600K .......... .......... .......... .......... .......... 51% 27.6M 3s 2024-09-17T23:48:12.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87650K .......... .......... .......... .......... .......... 52% 21.5M 3s 2024-09-17T23:48:12.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87700K .......... .......... .......... .......... .......... 52% 48.0M 3s 2024-09-17T23:48:12.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87750K .......... .......... .......... .......... .......... 52% 37.4M 3s 2024-09-17T23:48:12.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87800K .......... .......... .......... .......... .......... 52% 24.2M 3s 2024-09-17T23:48:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87850K .......... .......... .......... .......... .......... 52% 24.3M 3s 2024-09-17T23:48:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87900K .......... .......... .......... .......... .......... 52% 47.4M 3s 2024-09-17T23:48:12.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 87950K .......... .......... .......... .......... .......... 52% 23.4M 3s 2024-09-17T23:48:12.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88000K .......... .......... .......... .......... .......... 52% 24.3M 3s 2024-09-17T23:48:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88050K .......... .......... .......... .......... .......... 52% 38.0M 3s 2024-09-17T23:48:12.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88100K .......... .......... .......... .......... .......... 52% 25.2M 3s 2024-09-17T23:48:12.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88150K .......... .......... .......... .......... .......... 52% 42.0M 3s 2024-09-17T23:48:12.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88200K .......... .......... .......... .......... .......... 52% 24.7M 3s 2024-09-17T23:48:12.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88250K .......... .......... .......... .......... .......... 52% 34.0M 3s 2024-09-17T23:48:12.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88300K .......... .......... .......... .......... .......... 52% 26.0M 3s 2024-09-17T23:48:12.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88350K .......... .......... .......... .......... .......... 52% 21.8M 3s 2024-09-17T23:48:12.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88400K .......... .......... .......... .......... .......... 52% 24.2M 3s 2024-09-17T23:48:12.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88450K .......... .......... .......... .......... .......... 52% 47.5M 3s 2024-09-17T23:48:12.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88500K .......... .......... .......... .......... .......... 52% 31.3M 3s 2024-09-17T23:48:12.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88550K .......... .......... .......... .......... .......... 52% 28.4M 3s 2024-09-17T23:48:12.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88600K .......... .......... .......... .......... .......... 52% 25.1M 3s 2024-09-17T23:48:12.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88650K .......... .......... .......... .......... .......... 52% 46.9M 3s 2024-09-17T23:48:12.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88700K .......... .......... .......... .......... .......... 52% 24.3M 3s 2024-09-17T23:48:12.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88750K .......... .......... .......... .......... .......... 52% 23.6M 3s 2024-09-17T23:48:12.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88800K .......... .......... .......... .......... .......... 52% 40.2M 3s 2024-09-17T23:48:12.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88850K .......... .......... .......... .......... .......... 52% 21.9M 3s 2024-09-17T23:48:12.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88900K .......... .......... .......... .......... .......... 52% 26.9M 3s 2024-09-17T23:48:12.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 88950K .......... .......... .......... .......... .......... 52% 43.2M 3s 2024-09-17T23:48:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89000K .......... .......... .......... .......... .......... 52% 22.3M 3s 2024-09-17T23:48:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89050K .......... .......... .......... .......... .......... 52% 41.5M 3s 2024-09-17T23:48:12.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89100K .......... .......... .......... .......... .......... 52% 23.4M 3s 2024-09-17T23:48:12.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89150K .......... .......... .......... .......... .......... 52% 21.8M 3s 2024-09-17T23:48:12.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89200K .......... .......... .......... .......... .......... 52% 60.4M 3s 2024-09-17T23:48:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89250K .......... .......... .......... .......... .......... 52% 25.5M 3s 2024-09-17T23:48:12.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89300K .......... .......... .......... .......... .......... 53% 38.7M 3s 2024-09-17T23:48:12.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89350K .......... .......... .......... .......... .......... 53% 21.6M 3s 2024-09-17T23:48:12.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89400K .......... .......... .......... .......... .......... 53% 21.0M 3s 2024-09-17T23:48:12.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89450K .......... .......... .......... .......... .......... 53% 44.3M 3s 2024-09-17T23:48:12.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89500K .......... .......... .......... .......... .......... 53% 24.8M 3s 2024-09-17T23:48:12.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89550K .......... .......... .......... .......... .......... 53% 38.0M 3s 2024-09-17T23:48:12.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89600K .......... .......... .......... .......... .......... 53% 24.5M 3s 2024-09-17T23:48:12.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89650K .......... .......... .......... .......... .......... 53% 25.2M 3s 2024-09-17T23:48:12.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89700K .......... .......... .......... .......... .......... 53% 42.7M 3s 2024-09-17T23:48:12.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89750K .......... .......... .......... .......... .......... 53% 25.0M 3s 2024-09-17T23:48:12.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89800K .......... .......... .......... .......... .......... 53% 39.5M 3s 2024-09-17T23:48:12.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89850K .......... .......... .......... .......... .......... 53% 25.7M 3s 2024-09-17T23:48:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89900K .......... .......... .......... .......... .......... 53% 39.8M 3s 2024-09-17T23:48:12.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 89950K .......... .......... .......... .......... .......... 53% 16.2M 3s 2024-09-17T23:48:12.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90000K .......... .......... .......... .......... .......... 53% 42.1M 3s 2024-09-17T23:48:12.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90050K .......... .......... .......... .......... .......... 53% 23.2M 3s 2024-09-17T23:48:12.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90100K .......... .......... .......... .......... .......... 53% 34.9M 3s 2024-09-17T23:48:12.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90150K .......... .......... .......... .......... .......... 53% 20.6M 3s 2024-09-17T23:48:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90200K .......... .......... .......... .......... .......... 53% 40.5M 3s 2024-09-17T23:48:12.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90250K .......... .......... .......... .......... .......... 53% 24.7M 3s 2024-09-17T23:48:12.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90300K .......... .......... .......... .......... .......... 53% 35.0M 3s 2024-09-17T23:48:12.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90350K .......... .......... .......... .......... .......... 53% 20.2M 3s 2024-09-17T23:48:12.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90400K .......... .......... .......... .......... .......... 53% 25.3M 3s 2024-09-17T23:48:12.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90450K .......... .......... .......... .......... .......... 53% 42.3M 3s 2024-09-17T23:48:12.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90500K .......... .......... .......... .......... .......... 53% 24.3M 3s 2024-09-17T23:48:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90550K .......... .......... .......... .......... .......... 53% 10.5M 3s 2024-09-17T23:48:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90600K .......... .......... .......... .......... .......... 53% 115M 3s 2024-09-17T23:48:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90650K .......... .......... .......... .......... .......... 53% 48.1M 3s 2024-09-17T23:48:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90700K .......... .......... .......... .......... .......... 53% 43.9M 3s 2024-09-17T23:48:12.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90750K .......... .......... .......... .......... .......... 53% 19.2M 3s 2024-09-17T23:48:12.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90800K .......... .......... .......... .......... .......... 53% 25.6M 3s 2024-09-17T23:48:12.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90850K .......... .......... .......... .......... .......... 53% 38.1M 3s 2024-09-17T23:48:12.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90900K .......... .......... .......... .......... .......... 53% 21.5M 3s 2024-09-17T23:48:12.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 90950K .......... .......... .......... .......... .......... 53% 45.9M 3s 2024-09-17T23:48:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91000K .......... .......... .......... .......... .......... 54% 16.6M 3s 2024-09-17T23:48:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91050K .......... .......... .......... .......... .......... 54% 39.8M 3s 2024-09-17T23:48:12.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91100K .......... .......... .......... .......... .......... 54% 21.9M 3s 2024-09-17T23:48:12.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91150K .......... .......... .......... .......... .......... 54% 23.1M 3s 2024-09-17T23:48:12.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91200K .......... .......... .......... .......... .......... 54% 24.4M 3s 2024-09-17T23:48:12.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91250K .......... .......... .......... .......... .......... 54% 45.9M 3s 2024-09-17T23:48:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91300K .......... .......... .......... .......... .......... 54% 24.0M 3s 2024-09-17T23:48:12.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91350K .......... .......... .......... .......... .......... 54% 36.7M 3s 2024-09-17T23:48:12.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91400K .......... .......... .......... .......... .......... 54% 25.0M 3s 2024-09-17T23:48:12.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91450K .......... .......... .......... .......... .......... 54% 46.5M 3s 2024-09-17T23:48:12.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91500K .......... .......... .......... .......... .......... 54% 24.2M 3s 2024-09-17T23:48:12.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91550K .......... .......... .......... .......... .......... 54% 22.7M 3s 2024-09-17T23:48:12.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91600K .......... .......... .......... .......... .......... 54% 39.8M 3s 2024-09-17T23:48:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91650K .......... .......... .......... .......... .......... 54% 22.7M 3s 2024-09-17T23:48:12.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91700K .......... .......... .......... .......... .......... 54% 24.8M 3s 2024-09-17T23:48:12.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91750K .......... .......... .......... .......... .......... 54% 43.7M 3s 2024-09-17T23:48:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91800K .......... .......... .......... .......... .......... 54% 25.6M 3s 2024-09-17T23:48:12.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91850K .......... .......... .......... .......... .......... 54% 38.1M 3s 2024-09-17T23:48:12.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91900K .......... .......... .......... .......... .......... 54% 30.1M 3s 2024-09-17T23:48:12.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 91950K .......... .......... .......... .......... .......... 54% 20.2M 3s 2024-09-17T23:48:12.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92000K .......... .......... .......... .......... .......... 54% 47.8M 3s 2024-09-17T23:48:12.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92050K .......... .......... .......... .......... .......... 54% 24.6M 3s 2024-09-17T23:48:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92100K .......... .......... .......... .......... .......... 54% 39.4M 3s 2024-09-17T23:48:12.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92150K .......... .......... .......... .......... .......... 54% 22.0M 3s 2024-09-17T23:48:12.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92200K .......... .......... .......... .......... .......... 54% 25.9M 3s 2024-09-17T23:48:12.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92250K .......... .......... .......... .......... .......... 54% 49.8M 3s 2024-09-17T23:48:12.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92300K .......... .......... .......... .......... .......... 54% 23.2M 3s 2024-09-17T23:48:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92350K .......... .......... .......... .......... .......... 54% 23.0M 3s 2024-09-17T23:48:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92400K .......... .......... .......... .......... .......... 54% 34.8M 3s 2024-09-17T23:48:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92450K .......... .......... .......... .......... .......... 54% 25.6M 3s 2024-09-17T23:48:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92500K .......... .......... .......... .......... .......... 54% 43.1M 3s 2024-09-17T23:48:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92550K .......... .......... .......... .......... .......... 54% 23.1M 3s 2024-09-17T23:48:12.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92600K .......... .......... .......... .......... .......... 54% 38.2M 3s 2024-09-17T23:48:12.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92650K .......... .......... .......... .......... .......... 54% 25.1M 3s 2024-09-17T23:48:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92700K .......... .......... .......... .......... .......... 55% 44.0M 3s 2024-09-17T23:48:12.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92750K .......... .......... .......... .......... .......... 55% 16.4M 3s 2024-09-17T23:48:12.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92800K .......... .......... .......... .......... .......... 55% 41.5M 3s 2024-09-17T23:48:12.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92850K .......... .......... .......... .......... .......... 55% 25.6M 3s 2024-09-17T23:48:12.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92900K .......... .......... .......... .......... .......... 55% 39.8M 3s 2024-09-17T23:48:12.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 92950K .......... .......... .......... .......... .......... 55% 24.6M 3s 2024-09-17T23:48:12.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93000K .......... .......... .......... .......... .......... 55% 43.2M 3s 2024-09-17T23:48:12.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93050K .......... .......... .......... .......... .......... 55% 24.4M 3s 2024-09-17T23:48:12.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93100K .......... .......... .......... .......... .......... 55% 41.3M 3s 2024-09-17T23:48:12.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93150K .......... .......... .......... .......... .......... 55% 23.9M 3s 2024-09-17T23:48:12.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93200K .......... .......... .......... .......... .......... 55% 25.1M 3s 2024-09-17T23:48:12.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93250K .......... .......... .......... .......... .......... 55% 43.8M 3s 2024-09-17T23:48:12.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93300K .......... .......... .......... .......... .......... 55% 24.8M 3s 2024-09-17T23:48:12.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93350K .......... .......... .......... .......... .......... 55% 35.2M 3s 2024-09-17T23:48:12.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93400K .......... .......... .......... .......... .......... 55% 24.7M 3s 2024-09-17T23:48:12.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93450K .......... .......... .......... .......... .......... 55% 27.4M 3s 2024-09-17T23:48:12.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93500K .......... .......... .......... .......... .......... 55% 39.9M 3s 2024-09-17T23:48:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93550K .......... .......... .......... .......... .......... 55% 20.1M 3s 2024-09-17T23:48:12.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93600K .......... .......... .......... .......... .......... 55% 22.1M 3s 2024-09-17T23:48:12.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93650K .......... .......... .......... .......... .......... 55% 43.6M 3s 2024-09-17T23:48:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93700K .......... .......... .......... .......... .......... 55% 23.4M 3s 2024-09-17T23:48:12.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93750K .......... .......... .......... .......... .......... 55% 42.6M 3s 2024-09-17T23:48:12.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93800K .......... .......... .......... .......... .......... 55% 24.4M 3s 2024-09-17T23:48:12.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93850K .......... .......... .......... .......... .......... 55% 40.9M 3s 2024-09-17T23:48:12.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93900K .......... .......... .......... .......... .......... 55% 23.3M 3s 2024-09-17T23:48:12.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 93950K .......... .......... .......... .......... .......... 55% 22.8M 3s 2024-09-17T23:48:12.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94000K .......... .......... .......... .......... .......... 55% 26.7M 3s 2024-09-17T23:48:12.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94050K .......... .......... .......... .......... .......... 55% 42.5M 3s 2024-09-17T23:48:12.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94100K .......... .......... .......... .......... .......... 55% 24.8M 3s 2024-09-17T23:48:12.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94150K .......... .......... .......... .......... .......... 55% 38.1M 3s 2024-09-17T23:48:12.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94200K .......... .......... .......... .......... .......... 55% 24.4M 3s 2024-09-17T23:48:12.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94250K .......... .......... .......... .......... .......... 55% 37.6M 3s 2024-09-17T23:48:12.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94300K .......... .......... .......... .......... .......... 55% 22.4M 3s 2024-09-17T23:48:12.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94350K .......... .......... .......... .......... .......... 56% 21.7M 3s 2024-09-17T23:48:12.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94400K .......... .......... .......... .......... .......... 56% 38.3M 3s 2024-09-17T23:48:12.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94450K .......... .......... .......... .......... .......... 56% 20.9M 3s 2024-09-17T23:48:12.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94500K .......... .......... .......... .......... .......... 56% 22.2M 3s 2024-09-17T23:48:12.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94550K .......... .......... .......... .......... .......... 56% 36.6M 3s 2024-09-17T23:48:12.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94600K .......... .......... .......... .......... .......... 56% 23.4M 3s 2024-09-17T23:48:12.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94650K .......... .......... .......... .......... .......... 56% 39.3M 3s 2024-09-17T23:48:12.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94700K .......... .......... .......... .......... .......... 56% 24.3M 3s 2024-09-17T23:48:12.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94750K .......... .......... .......... .......... .......... 56% 20.1M 3s 2024-09-17T23:48:12.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94800K .......... .......... .......... .......... .......... 56% 43.5M 3s 2024-09-17T23:48:12.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94850K .......... .......... .......... .......... .......... 56% 23.3M 3s 2024-09-17T23:48:12.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94900K .......... .......... .......... .......... .......... 56% 33.4M 3s 2024-09-17T23:48:12.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 94950K .......... .......... .......... .......... .......... 56% 23.2M 3s 2024-09-17T23:48:12.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95000K .......... .......... .......... .......... .......... 56% 35.3M 3s 2024-09-17T23:48:12.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95050K .......... .......... .......... .......... .......... 56% 26.0M 3s 2024-09-17T23:48:12.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95100K .......... .......... .......... .......... .......... 56% 33.7M 3s 2024-09-17T23:48:12.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95150K .......... .......... .......... .......... .......... 56% 18.0M 3s 2024-09-17T23:48:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95200K .......... .......... .......... .......... .......... 56% 26.7M 3s 2024-09-17T23:48:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95250K .......... .......... .......... .......... .......... 56% 26.3M 3s 2024-09-17T23:48:12.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95300K .......... .......... .......... .......... .......... 56% 23.6M 3s 2024-09-17T23:48:12.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95350K .......... .......... .......... .......... .......... 56% 35.7M 3s 2024-09-17T23:48:12.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95400K .......... .......... .......... .......... .......... 56% 24.2M 3s 2024-09-17T23:48:12.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95450K .......... .......... .......... .......... .......... 56% 33.9M 3s 2024-09-17T23:48:12.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95500K .......... .......... .......... .......... .......... 56% 33.7M 3s 2024-09-17T23:48:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95550K .......... .......... .......... .......... .......... 56% 21.8M 3s 2024-09-17T23:48:12.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95600K .......... .......... .......... .......... .......... 56% 24.3M 3s 2024-09-17T23:48:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95650K .......... .......... .......... .......... .......... 56% 35.1M 3s 2024-09-17T23:48:12.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95700K .......... .......... .......... .......... .......... 56% 23.8M 3s 2024-09-17T23:48:12.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95750K .......... .......... .......... .......... .......... 56% 22.3M 3s 2024-09-17T23:48:12.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95800K .......... .......... .......... .......... .......... 56% 55.1M 3s 2024-09-17T23:48:12.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95850K .......... .......... .......... .......... .......... 56% 23.4M 3s 2024-09-17T23:48:12.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95900K .......... .......... .......... .......... .......... 56% 40.9M 3s 2024-09-17T23:48:12.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 95950K .......... .......... .......... .......... .......... 56% 23.1M 3s 2024-09-17T23:48:12.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96000K .......... .......... .......... .......... .......... 56% 35.0M 3s 2024-09-17T23:48:12.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96050K .......... .......... .......... .......... .......... 57% 32.1M 3s 2024-09-17T23:48:12.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96100K .......... .......... .......... .......... .......... 57% 23.9M 3s 2024-09-17T23:48:12.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96150K .......... .......... .......... .......... .......... 57% 36.0M 3s 2024-09-17T23:48:12.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96200K .......... .......... .......... .......... .......... 57% 23.9M 3s 2024-09-17T23:48:12.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96250K .......... .......... .......... .......... .......... 57% 23.3M 3s 2024-09-17T23:48:12.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96300K .......... .......... .......... .......... .......... 57% 51.3M 3s 2024-09-17T23:48:12.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96350K .......... .......... .......... .......... .......... 57% 24.0M 3s 2024-09-17T23:48:12.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96400K .......... .......... .......... .......... .......... 57% 31.8M 3s 2024-09-17T23:48:12.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96450K .......... .......... .......... .......... .......... 57% 34.7M 3s 2024-09-17T23:48:12.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96500K .......... .......... .......... .......... .......... 57% 21.2M 3s 2024-09-17T23:48:12.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96550K .......... .......... .......... .......... .......... 57% 45.4M 3s 2024-09-17T23:48:12.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96600K .......... .......... .......... .......... .......... 57% 25.2M 3s 2024-09-17T23:48:12.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96650K .......... .......... .......... .......... .......... 57% 26.6M 3s 2024-09-17T23:48:12.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96700K .......... .......... .......... .......... .......... 57% 30.5M 3s 2024-09-17T23:48:12.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96750K .......... .......... .......... .......... .......... 57% 22.3M 3s 2024-09-17T23:48:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96800K .......... .......... .......... .......... .......... 57% 25.4M 3s 2024-09-17T23:48:12.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96850K .......... .......... .......... .......... .......... 57% 44.5M 3s 2024-09-17T23:48:12.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96900K .......... .......... .......... .......... .......... 57% 24.5M 3s 2024-09-17T23:48:12.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 96950K .......... .......... .......... .......... .......... 57% 37.1M 3s 2024-09-17T23:48:12.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97000K .......... .......... .......... .......... .......... 57% 25.7M 3s 2024-09-17T23:48:12.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97050K .......... .......... .......... .......... .......... 57% 42.2M 3s 2024-09-17T23:48:12.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97100K .......... .......... .......... .......... .......... 57% 26.2M 3s 2024-09-17T23:48:12.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97150K .......... .......... .......... .......... .......... 57% 32.0M 3s 2024-09-17T23:48:12.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97200K .......... .......... .......... .......... .......... 57% 26.8M 2s 2024-09-17T23:48:12.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97250K .......... .......... .......... .......... .......... 57% 24.6M 2s 2024-09-17T23:48:12.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97300K .......... .......... .......... .......... .......... 57% 22.6M 2s 2024-09-17T23:48:12.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97350K .......... .......... .......... .......... .......... 57% 39.1M 2s 2024-09-17T23:48:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97400K .......... .......... .......... .......... .......... 57% 26.1M 2s 2024-09-17T23:48:12.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97450K .......... .......... .......... .......... .......... 57% 39.6M 2s 2024-09-17T23:48:12.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97500K .......... .......... .......... .......... .......... 57% 23.6M 2s 2024-09-17T23:48:12.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97550K .......... .......... .......... .......... .......... 57% 24.5M 2s 2024-09-17T23:48:12.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97600K .......... .......... .......... .......... .......... 57% 41.4M 2s 2024-09-17T23:48:12.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97650K .......... .......... .......... .......... .......... 57% 25.6M 2s 2024-09-17T23:48:12.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97700K .......... .......... .......... .......... .......... 57% 34.8M 2s 2024-09-17T23:48:12.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97750K .......... .......... .......... .......... .......... 58% 21.7M 2s 2024-09-17T23:48:12.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97800K .......... .......... .......... .......... .......... 58% 39.4M 2s 2024-09-17T23:48:12.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97850K .......... .......... .......... .......... .......... 58% 23.4M 2s 2024-09-17T23:48:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97900K .......... .......... .......... .......... .......... 58% 22.6M 2s 2024-09-17T23:48:12.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 97950K .......... .......... .......... .......... .......... 58% 27.4M 2s 2024-09-17T23:48:12.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98000K .......... .......... .......... .......... .......... 58% 34.4M 2s 2024-09-17T23:48:12.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98050K .......... .......... .......... .......... .......... 58% 24.0M 2s 2024-09-17T23:48:12.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98100K .......... .......... .......... .......... .......... 58% 42.7M 2s 2024-09-17T23:48:12.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98150K .......... .......... .......... .......... .......... 58% 23.6M 2s 2024-09-17T23:48:12.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98200K .......... .......... .......... .......... .......... 58% 40.1M 2s 2024-09-17T23:48:12.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98250K .......... .......... .......... .......... .......... 58% 24.6M 2s 2024-09-17T23:48:12.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98300K .......... .......... .......... .......... .......... 58% 55.7M 2s 2024-09-17T23:48:12.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98350K .......... .......... .......... .......... .......... 58% 21.2M 2s 2024-09-17T23:48:12.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98400K .......... .......... .......... .......... .......... 58% 25.5M 2s 2024-09-17T23:48:12.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98450K .......... .......... .......... .......... .......... 58% 21.1M 2s 2024-09-17T23:48:12.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98500K .......... .......... .......... .......... .......... 58% 41.5M 2s 2024-09-17T23:48:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98550K .......... .......... .......... .......... .......... 58% 20.3M 2s 2024-09-17T23:48:12.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98600K .......... .......... .......... .......... .......... 58% 39.0M 2s 2024-09-17T23:48:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98650K .......... .......... .......... .......... .......... 58% 23.7M 2s 2024-09-17T23:48:12.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98700K .......... .......... .......... .......... .......... 58% 37.0M 2s 2024-09-17T23:48:12.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98750K .......... .......... .......... .......... .......... 58% 21.3M 2s 2024-09-17T23:48:12.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98800K .......... .......... .......... .......... .......... 58% 21.3M 2s 2024-09-17T23:48:12.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98850K .......... .......... .......... .......... .......... 58% 43.4M 2s 2024-09-17T23:48:12.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98900K .......... .......... .......... .......... .......... 58% 28.0M 2s 2024-09-17T23:48:12.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 98950K .......... .......... .......... .......... .......... 58% 20.0M 2s 2024-09-17T23:48:12.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99000K .......... .......... .......... .......... .......... 58% 30.2M 2s 2024-09-17T23:48:12.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99050K .......... .......... .......... .......... .......... 58% 22.8M 2s 2024-09-17T23:48:12.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99100K .......... .......... .......... .......... .......... 58% 44.4M 2s 2024-09-17T23:48:12.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99150K .......... .......... .......... .......... .......... 58% 22.1M 2s 2024-09-17T23:48:12.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99200K .......... .......... .......... .......... .......... 58% 26.5M 2s 2024-09-17T23:48:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99250K .......... .......... .......... .......... .......... 58% 31.5M 2s 2024-09-17T23:48:12.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99300K .......... .......... .......... .......... .......... 58% 21.6M 2s 2024-09-17T23:48:12.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99350K .......... .......... .......... .......... .......... 58% 44.4M 2s 2024-09-17T23:48:12.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99400K .......... .......... .......... .......... .......... 58% 20.6M 2s 2024-09-17T23:48:12.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99450K .......... .......... .......... .......... .......... 59% 34.7M 2s 2024-09-17T23:48:12.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99500K .......... .......... .......... .......... .......... 59% 24.4M 2s 2024-09-17T23:48:12.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99550K .......... .......... .......... .......... .......... 59% 22.1M 2s 2024-09-17T23:48:12.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99600K .......... .......... .......... .......... .......... 59% 16.9M 2s 2024-09-17T23:48:12.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99650K .......... .......... .......... .......... .......... 59% 41.2M 2s 2024-09-17T23:48:12.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99700K .......... .......... .......... .......... .......... 59% 24.2M 2s 2024-09-17T23:48:12.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99750K .......... .......... .......... .......... .......... 59% 36.4M 2s 2024-09-17T23:48:12.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99800K .......... .......... .......... .......... .......... 59% 24.5M 2s 2024-09-17T23:48:12.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99850K .......... .......... .......... .......... .......... 59% 44.6M 2s 2024-09-17T23:48:12.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99900K .......... .......... .......... .......... .......... 59% 24.0M 2s 2024-09-17T23:48:12.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 99950K .......... .......... .......... .......... .......... 59% 24.1M 2s 2024-09-17T23:48:12.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100000K .......... .......... .......... .......... .......... 59% 38.2M 2s 2024-09-17T23:48:12.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100050K .......... .......... .......... .......... .......... 59% 25.6M 2s 2024-09-17T23:48:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100100K .......... .......... .......... .......... .......... 59% 22.8M 2s 2024-09-17T23:48:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100150K .......... .......... .......... .......... .......... 59% 49.3M 2s 2024-09-17T23:48:12.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100200K .......... .......... .......... .......... .......... 59% 35.6M 2s 2024-09-17T23:48:12.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100250K .......... .......... .......... .......... .......... 59% 25.0M 2s 2024-09-17T23:48:12.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100300K .......... .......... .......... .......... .......... 59% 24.3M 2s 2024-09-17T23:48:12.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100350K .......... .......... .......... .......... .......... 59% 25.2M 2s 2024-09-17T23:48:12.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100400K .......... .......... .......... .......... .......... 59% 39.1M 2s 2024-09-17T23:48:12.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100450K .......... .......... .......... .......... .......... 59% 26.9M 2s 2024-09-17T23:48:12.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100500K .......... .......... .......... .......... .......... 59% 36.1M 2s 2024-09-17T23:48:12.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100550K .......... .......... .......... .......... .......... 59% 24.0M 2s 2024-09-17T23:48:12.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100600K .......... .......... .......... .......... .......... 59% 50.8M 2s 2024-09-17T23:48:12.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100650K .......... .......... .......... .......... .......... 59% 23.9M 2s 2024-09-17T23:48:12.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100700K .......... .......... .......... .......... .......... 59% 36.5M 2s 2024-09-17T23:48:12.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100750K .......... .......... .......... .......... .......... 59% 23.1M 2s 2024-09-17T23:48:12.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100800K .......... .......... .......... .......... .......... 59% 24.4M 2s 2024-09-17T23:48:12.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100850K .......... .......... .......... .......... .......... 59% 21.9M 2s 2024-09-17T23:48:12.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100900K .......... .......... .......... .......... .......... 59% 46.5M 2s 2024-09-17T23:48:12.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:100950K .......... .......... .......... .......... .......... 59% 25.8M 2s 2024-09-17T23:48:12.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101000K .......... .......... .......... .......... .......... 59% 40.0M 2s 2024-09-17T23:48:12.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101050K .......... .......... .......... .......... .......... 59% 24.3M 2s 2024-09-17T23:48:12.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101100K .......... .......... .......... .......... .......... 60% 38.3M 2s 2024-09-17T23:48:12.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101150K .......... .......... .......... .......... .......... 60% 24.4M 2s 2024-09-17T23:48:12.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101200K .......... .......... .......... .......... .......... 60% 25.4M 2s 2024-09-17T23:48:12.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101250K .......... .......... .......... .......... .......... 60% 37.2M 2s 2024-09-17T23:48:12.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101300K .......... .......... .......... .......... .......... 60% 23.2M 2s 2024-09-17T23:48:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101350K .......... .......... .......... .......... .......... 60% 24.8M 2s 2024-09-17T23:48:12.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101400K .......... .......... .......... .......... .......... 60% 51.7M 2s 2024-09-17T23:48:12.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101450K .......... .......... .......... .......... .......... 60% 23.3M 2s 2024-09-17T23:48:12.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101500K .......... .......... .......... .......... .......... 60% 35.2M 2s 2024-09-17T23:48:12.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101550K .......... .......... .......... .......... .......... 60% 19.8M 2s 2024-09-17T23:48:12.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101600K .......... .......... .......... .......... .......... 60% 35.1M 2s 2024-09-17T23:48:12.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101650K .......... .......... .......... .......... .......... 60% 43.4M 2s 2024-09-17T23:48:12.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101700K .......... .......... .......... .......... .......... 60% 29.7M 2s 2024-09-17T23:48:12.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101750K .......... .......... .......... .......... .......... 60% 30.3M 2s 2024-09-17T23:48:12.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101800K .......... .......... .......... .......... .......... 60% 24.6M 2s 2024-09-17T23:48:12.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101850K .......... .......... .......... .......... .......... 60% 21.2M 2s 2024-09-17T23:48:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101900K .......... .......... .......... .......... .......... 60% 46.2M 2s 2024-09-17T23:48:12.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:101950K .......... .......... .......... .......... .......... 60% 21.8M 2s 2024-09-17T23:48:12.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102000K .......... .......... .......... .......... .......... 60% 21.7M 2s 2024-09-17T23:48:12.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102050K .......... .......... .......... .......... .......... 60% 42.6M 2s 2024-09-17T23:48:12.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102100K .......... .......... .......... .......... .......... 60% 22.9M 2s 2024-09-17T23:48:12.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102150K .......... .......... .......... .......... .......... 60% 45.8M 2s 2024-09-17T23:48:12.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102200K .......... .......... .......... .......... .......... 60% 25.4M 2s 2024-09-17T23:48:12.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102250K .......... .......... .......... .......... .......... 60% 37.8M 2s 2024-09-17T23:48:12.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102300K .......... .......... .......... .......... .......... 60% 24.3M 2s 2024-09-17T23:48:12.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102350K .......... .......... .......... .......... .......... 60% 24.7M 2s 2024-09-17T23:48:12.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102400K .......... .......... .......... .......... .......... 60% 22.4M 2s 2024-09-17T23:48:12.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102450K .......... .......... .......... .......... .......... 60% 45.6M 2s 2024-09-17T23:48:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102500K .......... .......... .......... .......... .......... 60% 33.2M 2s 2024-09-17T23:48:12.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102550K .......... .......... .......... .......... .......... 60% 27.9M 2s 2024-09-17T23:48:12.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102600K .......... .......... .......... .......... .......... 60% 23.8M 2s 2024-09-17T23:48:12.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102650K .......... .......... .......... .......... .......... 60% 46.8M 2s 2024-09-17T23:48:12.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102700K .......... .......... .......... .......... .......... 60% 22.6M 2s 2024-09-17T23:48:12.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102750K .......... .......... .......... .......... .......... 60% 21.4M 2s 2024-09-17T23:48:12.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102800K .......... .......... .......... .......... .......... 61% 35.5M 2s 2024-09-17T23:48:12.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102850K .......... .......... .......... .......... .......... 61% 29.7M 2s 2024-09-17T23:48:12.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102900K .......... .......... .......... .......... .......... 61% 31.8M 2s 2024-09-17T23:48:12.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:102950K .......... .......... .......... .......... .......... 61% 20.6M 2s 2024-09-17T23:48:12.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103000K .......... .......... .......... .......... .......... 61% 21.0M 2s 2024-09-17T23:48:12.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103050K .......... .......... .......... .......... .......... 61% 34.6M 2s 2024-09-17T23:48:12.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103100K .......... .......... .......... .......... .......... 61% 22.9M 2s 2024-09-17T23:48:12.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103150K .......... .......... .......... .......... .......... 61% 25.1M 2s 2024-09-17T23:48:12.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103200K .......... .......... .......... .......... .......... 61% 44.2M 2s 2024-09-17T23:48:12.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103250K .......... .......... .......... .......... .......... 61% 19.6M 2s 2024-09-17T23:48:12.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103300K .......... .......... .......... .......... .......... 61% 38.7M 2s 2024-09-17T23:48:12.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103350K .......... .......... .......... .......... .......... 61% 21.7M 2s 2024-09-17T23:48:12.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103400K .......... .......... .......... .......... .......... 61% 44.0M 2s 2024-09-17T23:48:12.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103450K .......... .......... .......... .......... .......... 61% 23.2M 2s 2024-09-17T23:48:12.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103500K .......... .......... .......... .......... .......... 61% 35.2M 2s 2024-09-17T23:48:12.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103550K .......... .......... .......... .......... .......... 61% 18.8M 2s 2024-09-17T23:48:12.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103600K .......... .......... .......... .......... .......... 61% 30.3M 2s 2024-09-17T23:48:12.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103650K .......... .......... .......... .......... .......... 61% 21.8M 2s 2024-09-17T23:48:12.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103700K .......... .......... .......... .......... .......... 61% 36.3M 2s 2024-09-17T23:48:12.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103750K .......... .......... .......... .......... .......... 61% 23.5M 2s 2024-09-17T23:48:12.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103800K .......... .......... .......... .......... .......... 61% 34.5M 2s 2024-09-17T23:48:12.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103850K .......... .......... .......... .......... .......... 61% 24.6M 2s 2024-09-17T23:48:12.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103900K .......... .......... .......... .......... .......... 61% 45.4M 2s 2024-09-17T23:48:12.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:103950K .......... .......... .......... .......... .......... 61% 15.8M 2s 2024-09-17T23:48:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104000K .......... .......... .......... .......... .......... 61% 23.9M 2s 2024-09-17T23:48:12.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104050K .......... .......... .......... .......... .......... 61% 29.3M 2s 2024-09-17T23:48:12.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104100K .......... .......... .......... .......... .......... 61% 39.1M 2s 2024-09-17T23:48:12.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104150K .......... .......... .......... .......... .......... 61% 20.8M 2s 2024-09-17T23:48:12.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104200K .......... .......... .......... .......... .......... 61% 37.3M 2s 2024-09-17T23:48:12.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104250K .......... .......... .......... .......... .......... 61% 23.5M 2s 2024-09-17T23:48:12.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104300K .......... .......... .......... .......... .......... 61% 39.7M 2s 2024-09-17T23:48:12.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104350K .......... .......... .......... .......... .......... 61% 23.4M 2s 2024-09-17T23:48:12.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104400K .......... .......... .......... .......... .......... 61% 25.4M 2s 2024-09-17T23:48:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104450K .......... .......... .......... .......... .......... 61% 45.1M 2s 2024-09-17T23:48:12.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104500K .......... .......... .......... .......... .......... 62% 25.1M 2s 2024-09-17T23:48:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104550K .......... .......... .......... .......... .......... 62% 38.2M 2s 2024-09-17T23:48:12.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104600K .......... .......... .......... .......... .......... 62% 24.9M 2s 2024-09-17T23:48:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104650K .......... .......... .......... .......... .......... 62% 22.8M 2s 2024-09-17T23:48:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104700K .......... .......... .......... .......... .......... 62% 49.4M 2s 2024-09-17T23:48:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104750K .......... .......... .......... .......... .......... 62% 30.0M 2s 2024-09-17T23:48:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104800K .......... .......... .......... .......... .......... 62% 20.6M 2s 2024-09-17T23:48:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104850K .......... .......... .......... .......... .......... 62% 35.0M 2s 2024-09-17T23:48:12.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104900K .......... .......... .......... .......... .......... 62% 25.5M 2s 2024-09-17T23:48:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:104950K .......... .......... .......... .......... .......... 62% 43.7M 2s 2024-09-17T23:48:12.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105000K .......... .......... .......... .......... .......... 62% 24.6M 2s 2024-09-17T23:48:12.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105050K .......... .......... .......... .......... .......... 62% 27.2M 2s 2024-09-17T23:48:12.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105100K .......... .......... .......... .......... .......... 62% 45.7M 2s 2024-09-17T23:48:12.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105150K .......... .......... .......... .......... .......... 62% 20.8M 2s 2024-09-17T23:48:12.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105200K .......... .......... .......... .......... .......... 62% 21.9M 2s 2024-09-17T23:48:12.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105250K .......... .......... .......... .......... .......... 62% 51.2M 2s 2024-09-17T23:48:12.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105300K .......... .......... .......... .......... .......... 62% 24.9M 2s 2024-09-17T23:48:12.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105350K .......... .......... .......... .......... .......... 62% 30.9M 2s 2024-09-17T23:48:12.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105400K .......... .......... .......... .......... .......... 62% 28.5M 2s 2024-09-17T23:48:12.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105450K .......... .......... .......... .......... .......... 62% 36.2M 2s 2024-09-17T23:48:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105500K .......... .......... .......... .......... .......... 62% 24.3M 2s 2024-09-17T23:48:12.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105550K .......... .......... .......... .......... .......... 62% 24.7M 2s 2024-09-17T23:48:12.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105600K .......... .......... .......... .......... .......... 62% 38.6M 2s 2024-09-17T23:48:12.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105650K .......... .......... .......... .......... .......... 62% 26.0M 2s 2024-09-17T23:48:12.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105700K .......... .......... .......... .......... .......... 62% 40.5M 2s 2024-09-17T23:48:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105750K .......... .......... .......... .......... .......... 62% 24.9M 2s 2024-09-17T23:48:12.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105800K .......... .......... .......... .......... .......... 62% 33.3M 2s 2024-09-17T23:48:12.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105850K .......... .......... .......... .......... .......... 62% 24.9M 2s 2024-09-17T23:48:12.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105900K .......... .......... .......... .......... .......... 62% 25.4M 2s 2024-09-17T23:48:12.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:105950K .......... .......... .......... .......... .......... 62% 24.6M 2s 2024-09-17T23:48:12.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106000K .......... .......... .......... .......... .......... 62% 41.2M 2s 2024-09-17T23:48:12.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106050K .......... .......... .......... .......... .......... 62% 25.6M 2s 2024-09-17T23:48:12.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106100K .......... .......... .......... .......... .......... 62% 34.1M 2s 2024-09-17T23:48:12.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106150K .......... .......... .......... .......... .......... 63% 20.9M 2s 2024-09-17T23:48:12.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106200K .......... .......... .......... .......... .......... 63% 41.4M 2s 2024-09-17T23:48:12.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106250K .......... .......... .......... .......... .......... 63% 22.2M 2s 2024-09-17T23:48:12.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106300K .......... .......... .......... .......... .......... 63% 36.6M 2s 2024-09-17T23:48:12.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106350K .......... .......... .......... .......... .......... 63% 25.4M 2s 2024-09-17T23:48:12.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106400K .......... .......... .......... .......... .......... 63% 28.8M 2s 2024-09-17T23:48:12.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106450K .......... .......... .......... .......... .......... 63% 21.1M 2s 2024-09-17T23:48:12.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106500K .......... .......... .......... .......... .......... 63% 39.8M 2s 2024-09-17T23:48:12.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106550K .......... .......... .......... .......... .......... 63% 25.8M 2s 2024-09-17T23:48:12.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106600K .......... .......... .......... .......... .......... 63% 38.3M 2s 2024-09-17T23:48:12.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106650K .......... .......... .......... .......... .......... 63% 24.5M 2s 2024-09-17T23:48:12.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106700K .......... .......... .......... .......... .......... 63% 42.4M 2s 2024-09-17T23:48:12.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106750K .......... .......... .......... .......... .......... 63% 24.0M 2s 2024-09-17T23:48:12.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106800K .......... .......... .......... .......... .......... 63% 25.7M 2s 2024-09-17T23:48:12.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106850K .......... .......... .......... .......... .......... 63% 37.4M 2s 2024-09-17T23:48:12.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106900K .......... .......... .......... .......... .......... 63% 23.3M 2s 2024-09-17T23:48:12.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:106950K .......... .......... .......... .......... .......... 63% 22.8M 2s 2024-09-17T23:48:12.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107000K .......... .......... .......... .......... .......... 63% 34.3M 2s 2024-09-17T23:48:12.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107050K .......... .......... .......... .......... .......... 63% 22.5M 2s 2024-09-17T23:48:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107100K .......... .......... .......... .......... .......... 63% 38.4M 2s 2024-09-17T23:48:12.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107150K .......... .......... .......... .......... .......... 63% 20.5M 2s 2024-09-17T23:48:12.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107200K .......... .......... .......... .......... .......... 63% 24.3M 2s 2024-09-17T23:48:12.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107250K .......... .......... .......... .......... .......... 63% 47.3M 2s 2024-09-17T23:48:12.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107300K .......... .......... .......... .......... .......... 63% 20.5M 2s 2024-09-17T23:48:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107350K .......... .......... .......... .......... .......... 63% 32.5M 2s 2024-09-17T23:48:12.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107400K .......... .......... .......... .......... .......... 63% 25.9M 2s 2024-09-17T23:48:12.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107450K .......... .......... .......... .......... .......... 63% 41.1M 2s 2024-09-17T23:48:12.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107500K .......... .......... .......... .......... .......... 63% 23.6M 2s 2024-09-17T23:48:12.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107550K .......... .......... .......... .......... .......... 63% 18.7M 2s 2024-09-17T23:48:12.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107600K .......... .......... .......... .......... .......... 63% 23.5M 2s 2024-09-17T23:48:12.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107650K .......... .......... .......... .......... .......... 63% 40.5M 2s 2024-09-17T23:48:12.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107700K .......... .......... .......... .......... .......... 63% 21.0M 2s 2024-09-17T23:48:12.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107750K .......... .......... .......... .......... .......... 63% 41.9M 2s 2024-09-17T23:48:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107800K .......... .......... .......... .......... .......... 63% 24.5M 2s 2024-09-17T23:48:12.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107850K .......... .......... .......... .......... .......... 64% 21.9M 2s 2024-09-17T23:48:12.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107900K .......... .......... .......... .......... .......... 64% 38.1M 2s 2024-09-17T23:48:12.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:107950K .......... .......... .......... .......... .......... 64% 22.5M 2s 2024-09-17T23:48:12.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108000K .......... .......... .......... .......... .......... 64% 36.2M 2s 2024-09-17T23:48:12.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108050K .......... .......... .......... .......... .......... 64% 24.3M 2s 2024-09-17T23:48:12.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108100K .......... .......... .......... .......... .......... 64% 34.9M 2s 2024-09-17T23:48:12.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108150K .......... .......... .......... .......... .......... 64% 22.8M 2s 2024-09-17T23:48:12.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108200K .......... .......... .......... .......... .......... 64% 24.0M 2s 2024-09-17T23:48:12.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108250K .......... .......... .......... .......... .......... 64% 23.4M 2s 2024-09-17T23:48:12.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108300K .......... .......... .......... .......... .......... 64% 21.5M 2s 2024-09-17T23:48:12.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108350K .......... .......... .......... .......... .......... 64% 23.9M 2s 2024-09-17T23:48:12.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108400K .......... .......... .......... .......... .......... 64% 39.2M 2s 2024-09-17T23:48:12.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108450K .......... .......... .......... .......... .......... 64% 25.2M 2s 2024-09-17T23:48:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108500K .......... .......... .......... .......... .......... 64% 44.8M 2s 2024-09-17T23:48:12.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108550K .......... .......... .......... .......... .......... 64% 26.5M 2s 2024-09-17T23:48:12.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108600K .......... .......... .......... .......... .......... 64% 30.4M 2s 2024-09-17T23:48:12.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108650K .......... .......... .......... .......... .......... 64% 24.4M 2s 2024-09-17T23:48:12.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108700K .......... .......... .......... .......... .......... 64% 25.3M 2s 2024-09-17T23:48:12.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108750K .......... .......... .......... .......... .......... 64% 25.0M 2s 2024-09-17T23:48:12.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108800K .......... .......... .......... .......... .......... 64% 38.2M 2s 2024-09-17T23:48:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108850K .......... .......... .......... .......... .......... 64% 25.5M 2s 2024-09-17T23:48:12.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108900K .......... .......... .......... .......... .......... 64% 38.5M 2s 2024-09-17T23:48:12.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:108950K .......... .......... .......... .......... .......... 64% 25.2M 2s 2024-09-17T23:48:12.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109000K .......... .......... .......... .......... .......... 64% 42.4M 2s 2024-09-17T23:48:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109050K .......... .......... .......... .......... .......... 64% 24.6M 2s 2024-09-17T23:48:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109100K .......... .......... .......... .......... .......... 64% 43.0M 2s 2024-09-17T23:48:12.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109150K .......... .......... .......... .......... .......... 64% 17.3M 2s 2024-09-17T23:48:12.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109200K .......... .......... .......... .......... .......... 64% 31.8M 2s 2024-09-17T23:48:12.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109250K .......... .......... .......... .......... .......... 64% 27.5M 2s 2024-09-17T23:48:12.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109300K .......... .......... .......... .......... .......... 64% 41.6M 2s 2024-09-17T23:48:12.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109350K .......... .......... .......... .......... .......... 64% 27.2M 2s 2024-09-17T23:48:12.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109400K .......... .......... .......... .......... .......... 64% 34.2M 2s 2024-09-17T23:48:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109450K .......... .......... .......... .......... .......... 64% 22.6M 2s 2024-09-17T23:48:12.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109500K .......... .......... .......... .......... .......... 64% 48.3M 2s 2024-09-17T23:48:12.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109550K .......... .......... .......... .......... .......... 65% 23.5M 2s 2024-09-17T23:48:12.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109600K .......... .......... .......... .......... .......... 65% 24.7M 2s 2024-09-17T23:48:12.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109650K .......... .......... .......... .......... .......... 65% 31.2M 2s 2024-09-17T23:48:12.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109700K .......... .......... .......... .......... .......... 65% 28.8M 2s 2024-09-17T23:48:12.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109750K .......... .......... .......... .......... .......... 65% 35.0M 2s 2024-09-17T23:48:12.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109800K .......... .......... .......... .......... .......... 65% 30.8M 2s 2024-09-17T23:48:12.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109850K .......... .......... .......... .......... .......... 65% 25.9M 2s 2024-09-17T23:48:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109900K .......... .......... .......... .......... .......... 65% 30.6M 2s 2024-09-17T23:48:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:109950K .......... .......... .......... .......... .......... 65% 23.8M 2s 2024-09-17T23:48:12.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110000K .......... .......... .......... .......... .......... 65% 25.9M 2s 2024-09-17T23:48:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110050K .......... .......... .......... .......... .......... 65% 41.9M 2s 2024-09-17T23:48:12.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110100K .......... .......... .......... .......... .......... 65% 24.3M 2s 2024-09-17T23:48:12.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110150K .......... .......... .......... .......... .......... 65% 39.1M 2s 2024-09-17T23:48:12.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110200K .......... .......... .......... .......... .......... 65% 25.2M 2s 2024-09-17T23:48:12.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110250K .......... .......... .......... .......... .......... 65% 43.8M 2s 2024-09-17T23:48:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110300K .......... .......... .......... .......... .......... 65% 24.7M 2s 2024-09-17T23:48:12.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110350K .......... .......... .......... .......... .......... 65% 21.2M 2s 2024-09-17T23:48:12.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110400K .......... .......... .......... .......... .......... 65% 26.6M 2s 2024-09-17T23:48:12.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110450K .......... .......... .......... .......... .......... 65% 22.5M 2s 2024-09-17T23:48:12.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110500K .......... .......... .......... .......... .......... 65% 24.9M 2s 2024-09-17T23:48:12.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110550K .......... .......... .......... .......... .......... 65% 44.6M 2s 2024-09-17T23:48:12.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110600K .......... .......... .......... .......... .......... 65% 24.0M 2s 2024-09-17T23:48:12.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110650K .......... .......... .......... .......... .......... 65% 41.4M 2s 2024-09-17T23:48:12.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110700K .......... .......... .......... .......... .......... 65% 24.5M 2s 2024-09-17T23:48:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110750K .......... .......... .......... .......... .......... 65% 23.6M 2s 2024-09-17T23:48:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110800K .......... .......... .......... .......... .......... 65% 42.3M 2s 2024-09-17T23:48:12.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110850K .......... .......... .......... .......... .......... 65% 26.6M 2s 2024-09-17T23:48:12.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110900K .......... .......... .......... .......... .......... 65% 34.8M 2s 2024-09-17T23:48:12.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:110950K .......... .......... .......... .......... .......... 65% 26.0M 2s 2024-09-17T23:48:12.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111000K .......... .......... .......... .......... .......... 65% 24.2M 2s 2024-09-17T23:48:13.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111050K .......... .......... .......... .......... .......... 65% 41.5M 2s 2024-09-17T23:48:13.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111100K .......... .......... .......... .......... .......... 65% 23.9M 2s 2024-09-17T23:48:13.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111150K .......... .......... .......... .......... .......... 65% 24.4M 2s 2024-09-17T23:48:13.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111200K .......... .......... .......... .......... .......... 65% 31.8M 2s 2024-09-17T23:48:13.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111250K .......... .......... .......... .......... .......... 66% 22.5M 2s 2024-09-17T23:48:13.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111300K .......... .......... .......... .......... .......... 66% 42.4M 2s 2024-09-17T23:48:13.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111350K .......... .......... .......... .......... .......... 66% 22.0M 2s 2024-09-17T23:48:13.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111400K .......... .......... .......... .......... .......... 66% 34.2M 2s 2024-09-17T23:48:13.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111450K .......... .......... .......... .......... .......... 66% 24.2M 2s 2024-09-17T23:48:13.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111500K .......... .......... .......... .......... .......... 66% 21.6M 2s 2024-09-17T23:48:13.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111550K .......... .......... .......... .......... .......... 66% 21.9M 2s 2024-09-17T23:48:13.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111600K .......... .......... .......... .......... .......... 66% 36.0M 2s 2024-09-17T23:48:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111650K .......... .......... .......... .......... .......... 66% 25.3M 2s 2024-09-17T23:48:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111700K .......... .......... .......... .......... .......... 66% 38.8M 2s 2024-09-17T23:48:13.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111750K .......... .......... .......... .......... .......... 66% 24.0M 2s 2024-09-17T23:48:13.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111800K .......... .......... .......... .......... .......... 66% 33.2M 2s 2024-09-17T23:48:13.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111850K .......... .......... .......... .......... .......... 66% 21.0M 2s 2024-09-17T23:48:13.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111900K .......... .......... .......... .......... .......... 66% 38.3M 2s 2024-09-17T23:48:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:111950K .......... .......... .......... .......... .......... 66% 22.6M 2s 2024-09-17T23:48:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112000K .......... .......... .......... .......... .......... 66% 21.8M 2s 2024-09-17T23:48:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112050K .......... .......... .......... .......... .......... 66% 22.6M 2s 2024-09-17T23:48:13.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112100K .......... .......... .......... .......... .......... 66% 65.9M 2s 2024-09-17T23:48:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112150K .......... .......... .......... .......... .......... 66% 18.9M 2s 2024-09-17T23:48:13.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112200K .......... .......... .......... .......... .......... 66% 34.1M 2s 2024-09-17T23:48:13.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112250K .......... .......... .......... .......... .......... 66% 25.1M 2s 2024-09-17T23:48:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112300K .......... .......... .......... .......... .......... 66% 41.5M 2s 2024-09-17T23:48:13.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112350K .......... .......... .......... .......... .......... 66% 19.9M 2s 2024-09-17T23:48:13.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112400K .......... .......... .......... .......... .......... 66% 25.4M 2s 2024-09-17T23:48:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112450K .......... .......... .......... .......... .......... 66% 37.2M 2s 2024-09-17T23:48:13.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112500K .......... .......... .......... .......... .......... 66% 24.7M 2s 2024-09-17T23:48:13.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112550K .......... .......... .......... .......... .......... 66% 34.8M 2s 2024-09-17T23:48:13.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112600K .......... .......... .......... .......... .......... 66% 16.5M 2s 2024-09-17T23:48:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112650K .......... .......... .......... .......... .......... 66% 36.4M 2s 2024-09-17T23:48:13.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112700K .......... .......... .......... .......... .......... 66% 22.5M 2s 2024-09-17T23:48:13.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112750K .......... .......... .......... .......... .......... 66% 23.4M 2s 2024-09-17T23:48:13.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112800K .......... .......... .......... .......... .......... 66% 24.8M 2s 2024-09-17T23:48:13.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112850K .......... .......... .......... .......... .......... 66% 55.0M 2s 2024-09-17T23:48:13.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112900K .......... .......... .......... .......... .......... 67% 22.1M 2s 2024-09-17T23:48:13.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:112950K .......... .......... .......... .......... .......... 67% 35.0M 2s 2024-09-17T23:48:13.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113000K .......... .......... .......... .......... .......... 67% 24.9M 2s 2024-09-17T23:48:13.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113050K .......... .......... .......... .......... .......... 67% 47.6M 2s 2024-09-17T23:48:13.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113100K .......... .......... .......... .......... .......... 67% 24.7M 2s 2024-09-17T23:48:13.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113150K .......... .......... .......... .......... .......... 67% 26.7M 2s 2024-09-17T23:48:13.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113200K .......... .......... .......... .......... .......... 67% 31.9M 2s 2024-09-17T23:48:13.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113250K .......... .......... .......... .......... .......... 67% 24.3M 2s 2024-09-17T23:48:13.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113300K .......... .......... .......... .......... .......... 67% 24.2M 2s 2024-09-17T23:48:13.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113350K .......... .......... .......... .......... .......... 67% 44.9M 2s 2024-09-17T23:48:13.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113400K .......... .......... .......... .......... .......... 67% 23.4M 2s 2024-09-17T23:48:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113450K .......... .......... .......... .......... .......... 67% 39.6M 2s 2024-09-17T23:48:13.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113500K .......... .......... .......... .......... .......... 67% 24.3M 2s 2024-09-17T23:48:13.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113550K .......... .......... .......... .......... .......... 67% 24.0M 2s 2024-09-17T23:48:13.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113600K .......... .......... .......... .......... .......... 67% 56.0M 2s 2024-09-17T23:48:13.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113650K .......... .......... .......... .......... .......... 67% 24.4M 2s 2024-09-17T23:48:13.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113700K .......... .......... .......... .......... .......... 67% 34.7M 2s 2024-09-17T23:48:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113750K .......... .......... .......... .......... .......... 67% 24.0M 2s 2024-09-17T23:48:13.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113800K .......... .......... .......... .......... .......... 67% 23.6M 2s 2024-09-17T23:48:13.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113850K .......... .......... .......... .......... .......... 67% 55.9M 2s 2024-09-17T23:48:13.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113900K .......... .......... .......... .......... .......... 67% 25.5M 2s 2024-09-17T23:48:13.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:113950K .......... .......... .......... .......... .......... 67% 22.8M 2s 2024-09-17T23:48:13.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114000K .......... .......... .......... .......... .......... 67% 31.8M 2s 2024-09-17T23:48:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114050K .......... .......... .......... .......... .......... 67% 30.4M 2s 2024-09-17T23:48:13.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114100K .......... .......... .......... .......... .......... 67% 35.8M 2s 2024-09-17T23:48:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114150K .......... .......... .......... .......... .......... 67% 23.2M 2s 2024-09-17T23:48:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114200K .......... .......... .......... .......... .......... 67% 39.0M 2s 2024-09-17T23:48:13.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114250K .......... .......... .......... .......... .......... 67% 24.7M 2s 2024-09-17T23:48:13.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114300K .......... .......... .......... .......... .......... 67% 23.6M 2s 2024-09-17T23:48:13.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114350K .......... .......... .......... .......... .......... 67% 24.4M 2s 2024-09-17T23:48:13.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114400K .......... .......... .......... .......... .......... 67% 39.9M 2s 2024-09-17T23:48:13.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114450K .......... .......... .......... .......... .......... 67% 26.4M 2s 2024-09-17T23:48:13.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114500K .......... .......... .......... .......... .......... 67% 40.1M 2s 2024-09-17T23:48:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114550K .......... .......... .......... .......... .......... 67% 24.1M 2s 2024-09-17T23:48:13.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114600K .......... .......... .......... .......... .......... 68% 44.8M 2s 2024-09-17T23:48:13.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114650K .......... .......... .......... .......... .......... 68% 20.8M 2s 2024-09-17T23:48:13.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114700K .......... .......... .......... .......... .......... 68% 34.6M 2s 2024-09-17T23:48:13.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114750K .......... .......... .......... .......... .......... 68% 19.9M 2s 2024-09-17T23:48:13.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114800K .......... .......... .......... .......... .......... 68% 24.8M 2s 2024-09-17T23:48:13.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114850K .......... .......... .......... .......... .......... 68% 23.6M 2s 2024-09-17T23:48:13.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114900K .......... .......... .......... .......... .......... 68% 42.6M 2s 2024-09-17T23:48:13.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:114950K .......... .......... .......... .......... .......... 68% 24.8M 2s 2024-09-17T23:48:13.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115000K .......... .......... .......... .......... .......... 68% 39.3M 2s 2024-09-17T23:48:13.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115050K .......... .......... .......... .......... .......... 68% 25.4M 2s 2024-09-17T23:48:13.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115100K .......... .......... .......... .......... .......... 68% 42.4M 2s 2024-09-17T23:48:13.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115150K .......... .......... .......... .......... .......... 68% 22.5M 2s 2024-09-17T23:48:13.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115200K .......... .......... .......... .......... .......... 68% 25.2M 2s 2024-09-17T23:48:13.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115250K .......... .......... .......... .......... .......... 68% 39.7M 2s 2024-09-17T23:48:13.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115300K .......... .......... .......... .......... .......... 68% 22.6M 2s 2024-09-17T23:48:13.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115350K .......... .......... .......... .......... .......... 68% 43.6M 2s 2024-09-17T23:48:13.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115400K .......... .......... .......... .......... .......... 68% 23.0M 2s 2024-09-17T23:48:13.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115450K .......... .......... .......... .......... .......... 68% 22.8M 2s 2024-09-17T23:48:13.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115500K .......... .......... .......... .......... .......... 68% 32.9M 2s 2024-09-17T23:48:13.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115550K .......... .......... .......... .......... .......... 68% 21.4M 2s 2024-09-17T23:48:13.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115600K .......... .......... .......... .......... .......... 68% 23.2M 2s 2024-09-17T23:48:13.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115650K .......... .......... .......... .......... .......... 68% 46.3M 2s 2024-09-17T23:48:13.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115700K .......... .......... .......... .......... .......... 68% 22.4M 2s 2024-09-17T23:48:13.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115750K .......... .......... .......... .......... .......... 68% 38.3M 2s 2024-09-17T23:48:13.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115800K .......... .......... .......... .......... .......... 68% 21.5M 2s 2024-09-17T23:48:13.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115850K .......... .......... .......... .......... .......... 68% 43.1M 2s 2024-09-17T23:48:13.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115900K .......... .......... .......... .......... .......... 68% 22.0M 2s 2024-09-17T23:48:13.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:115950K .......... .......... .......... .......... .......... 68% 23.6M 2s 2024-09-17T23:48:13.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116000K .......... .......... .......... .......... .......... 68% 23.7M 2s 2024-09-17T23:48:13.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116050K .......... .......... .......... .......... .......... 68% 37.9M 2s 2024-09-17T23:48:13.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116100K .......... .......... .......... .......... .......... 68% 18.3M 2s 2024-09-17T23:48:13.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116150K .......... .......... .......... .......... .......... 68% 44.8M 2s 2024-09-17T23:48:13.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116200K .......... .......... .......... .......... .......... 68% 22.5M 2s 2024-09-17T23:48:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116250K .......... .......... .......... .......... .......... 68% 38.2M 2s 2024-09-17T23:48:13.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116300K .......... .......... .......... .......... .......... 69% 23.0M 2s 2024-09-17T23:48:13.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116350K .......... .......... .......... .......... .......... 69% 22.4M 2s 2024-09-17T23:48:13.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116400K .......... .......... .......... .......... .......... 69% 42.8M 2s 2024-09-17T23:48:13.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116450K .......... .......... .......... .......... .......... 69% 26.3M 2s 2024-09-17T23:48:13.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116500K .......... .......... .......... .......... .......... 69% 29.8M 2s 2024-09-17T23:48:13.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116550K .......... .......... .......... .......... .......... 69% 23.3M 2s 2024-09-17T23:48:13.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116600K .......... .......... .......... .......... .......... 69% 23.0M 2s 2024-09-17T23:48:13.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116650K .......... .......... .......... .......... .......... 69% 33.6M 2s 2024-09-17T23:48:13.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116700K .......... .......... .......... .......... .......... 69% 23.5M 2s 2024-09-17T23:48:13.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116750K .......... .......... .......... .......... .......... 69% 24.4M 2s 2024-09-17T23:48:13.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116800K .......... .......... .......... .......... .......... 69% 34.7M 2s 2024-09-17T23:48:13.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116850K .......... .......... .......... .......... .......... 69% 22.5M 2s 2024-09-17T23:48:13.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116900K .......... .......... .......... .......... .......... 69% 30.1M 2s 2024-09-17T23:48:13.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:116950K .......... .......... .......... .......... .......... 69% 23.6M 2s 2024-09-17T23:48:13.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117000K .......... .......... .......... .......... .......... 69% 38.0M 2s 2024-09-17T23:48:13.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117050K .......... .......... .......... .......... .......... 69% 19.9M 2s 2024-09-17T23:48:13.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117100K .......... .......... .......... .......... .......... 69% 23.8M 2s 2024-09-17T23:48:13.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117150K .......... .......... .......... .......... .......... 69% 26.3M 2s 2024-09-17T23:48:13.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117200K .......... .......... .......... .......... .......... 69% 38.5M 2s 2024-09-17T23:48:13.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117250K .......... .......... .......... .......... .......... 69% 22.6M 2s 2024-09-17T23:48:13.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117300K .......... .......... .......... .......... .......... 69% 36.3M 2s 2024-09-17T23:48:13.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117350K .......... .......... .......... .......... .......... 69% 39.9M 2s 2024-09-17T23:48:13.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117400K .......... .......... .......... .......... .......... 69% 30.1M 2s 2024-09-17T23:48:13.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117450K .......... .......... .......... .......... .......... 69% 22.3M 2s 2024-09-17T23:48:13.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117500K .......... .......... .......... .......... .......... 69% 38.8M 2s 2024-09-17T23:48:13.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117550K .......... .......... .......... .......... .......... 69% 23.4M 2s 2024-09-17T23:48:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117600K .......... .......... .......... .......... .......... 69% 22.2M 2s 2024-09-17T23:48:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117650K .......... .......... .......... .......... .......... 69% 60.5M 2s 2024-09-17T23:48:13.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117700K .......... .......... .......... .......... .......... 69% 25.6M 2s 2024-09-17T23:48:13.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117750K .......... .......... .......... .......... .......... 69% 32.4M 2s 2024-09-17T23:48:13.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117800K .......... .......... .......... .......... .......... 69% 21.0M 2s 2024-09-17T23:48:13.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117850K .......... .......... .......... .......... .......... 69% 30.2M 2s 2024-09-17T23:48:13.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117900K .......... .......... .......... .......... .......... 69% 44.2M 2s 2024-09-17T23:48:13.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:117950K .......... .......... .......... .......... .......... 70% 24.2M 2s 2024-09-17T23:48:13.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118000K .......... .......... .......... .......... .......... 70% 25.9M 2s 2024-09-17T23:48:13.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118050K .......... .......... .......... .......... .......... 70% 42.9M 2s 2024-09-17T23:48:13.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118100K .......... .......... .......... .......... .......... 70% 23.5M 2s 2024-09-17T23:48:13.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118150K .......... .......... .......... .......... .......... 70% 44.7M 2s 2024-09-17T23:48:13.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118200K .......... .......... .......... .......... .......... 70% 24.5M 2s 2024-09-17T23:48:13.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118250K .......... .......... .......... .......... .......... 70% 38.9M 2s 2024-09-17T23:48:13.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118300K .......... .......... .......... .......... .......... 70% 21.6M 2s 2024-09-17T23:48:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118350K .......... .......... .......... .......... .......... 70% 23.8M 2s 2024-09-17T23:48:13.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118400K .......... .......... .......... .......... .......... 70% 23.8M 2s 2024-09-17T23:48:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118450K .......... .......... .......... .......... .......... 70% 43.9M 2s 2024-09-17T23:48:13.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118500K .......... .......... .......... .......... .......... 70% 38.8M 2s 2024-09-17T23:48:13.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118550K .......... .......... .......... .......... .......... 70% 26.2M 2s 2024-09-17T23:48:13.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118600K .......... .......... .......... .......... .......... 70% 35.0M 2s 2024-09-17T23:48:13.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118650K .......... .......... .......... .......... .......... 70% 30.7M 2s 2024-09-17T23:48:13.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118700K .......... .......... .......... .......... .......... 70% 22.6M 2s 2024-09-17T23:48:13.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118750K .......... .......... .......... .......... .......... 70% 23.7M 2s 2024-09-17T23:48:13.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118800K .......... .......... .......... .......... .......... 70% 38.8M 2s 2024-09-17T23:48:13.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118850K .......... .......... .......... .......... .......... 70% 24.2M 2s 2024-09-17T23:48:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118900K .......... .......... .......... .......... .......... 70% 21.1M 2s 2024-09-17T23:48:13.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:118950K .......... .......... .......... .......... .......... 70% 33.6M 2s 2024-09-17T23:48:13.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119000K .......... .......... .......... .......... .......... 70% 20.1M 2s 2024-09-17T23:48:13.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119050K .......... .......... .......... .......... .......... 70% 42.2M 2s 2024-09-17T23:48:13.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119100K .......... .......... .......... .......... .......... 70% 9.93M 2s 2024-09-17T23:48:13.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119150K .......... .......... .......... .......... .......... 70% 82.3M 2s 2024-09-17T23:48:13.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119200K .......... .......... .......... .......... .......... 70% 127M 2s 2024-09-17T23:48:13.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119250K .......... .......... .......... .......... .......... 70% 40.8M 2s 2024-09-17T23:48:13.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119300K .......... .......... .......... .......... .......... 70% 37.8M 2s 2024-09-17T23:48:13.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119350K .......... .......... .......... .......... .......... 70% 24.9M 2s 2024-09-17T23:48:13.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119400K .......... .......... .......... .......... .......... 70% 23.5M 2s 2024-09-17T23:48:13.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119450K .......... .......... .......... .......... .......... 70% 45.3M 2s 2024-09-17T23:48:13.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119500K .......... .......... .......... .......... .......... 70% 24.7M 2s 2024-09-17T23:48:13.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119550K .......... .......... .......... .......... .......... 70% 23.7M 2s 2024-09-17T23:48:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119600K .......... .......... .......... .......... .......... 70% 39.9M 2s 2024-09-17T23:48:13.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119650K .......... .......... .......... .......... .......... 71% 23.6M 2s 2024-09-17T23:48:13.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119700K .......... .......... .......... .......... .......... 71% 40.2M 2s 2024-09-17T23:48:13.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119750K .......... .......... .......... .......... .......... 71% 20.8M 2s 2024-09-17T23:48:13.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119800K .......... .......... .......... .......... .......... 71% 37.8M 2s 2024-09-17T23:48:13.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119850K .......... .......... .......... .......... .......... 71% 22.9M 2s 2024-09-17T23:48:13.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119900K .......... .......... .......... .......... .......... 71% 44.9M 2s 2024-09-17T23:48:13.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:119950K .......... .......... .......... .......... .......... 71% 14.5M 2s 2024-09-17T23:48:13.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120000K .......... .......... .......... .......... .......... 71% 45.5M 2s 2024-09-17T23:48:13.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120050K .......... .......... .......... .......... .......... 71% 23.4M 2s 2024-09-17T23:48:13.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120100K .......... .......... .......... .......... .......... 71% 36.6M 2s 2024-09-17T23:48:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120150K .......... .......... .......... .......... .......... 71% 19.4M 2s 2024-09-17T23:48:13.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120200K .......... .......... .......... .......... .......... 71% 46.1M 2s 2024-09-17T23:48:13.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120250K .......... .......... .......... .......... .......... 71% 24.7M 2s 2024-09-17T23:48:13.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120300K .......... .......... .......... .......... .......... 71% 40.7M 2s 2024-09-17T23:48:13.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120350K .......... .......... .......... .......... .......... 71% 23.8M 2s 2024-09-17T23:48:13.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120400K .......... .......... .......... .......... .......... 71% 20.6M 2s 2024-09-17T23:48:13.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120450K .......... .......... .......... .......... .......... 71% 33.8M 2s 2024-09-17T23:48:13.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120500K .......... .......... .......... .......... .......... 71% 24.6M 2s 2024-09-17T23:48:13.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120550K .......... .......... .......... .......... .......... 71% 21.7M 2s 2024-09-17T23:48:13.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120600K .......... .......... .......... .......... .......... 71% 37.1M 2s 2024-09-17T23:48:13.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120650K .......... .......... .......... .......... .......... 71% 23.7M 2s 2024-09-17T23:48:13.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120700K .......... .......... .......... .......... .......... 71% 36.2M 2s 2024-09-17T23:48:13.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120750K .......... .......... .......... .......... .......... 71% 22.5M 2s 2024-09-17T23:48:13.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120800K .......... .......... .......... .......... .......... 71% 26.9M 2s 2024-09-17T23:48:13.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120850K .......... .......... .......... .......... .......... 71% 31.4M 2s 2024-09-17T23:48:13.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120900K .......... .......... .......... .......... .......... 71% 26.1M 2s 2024-09-17T23:48:13.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:120950K .......... .......... .......... .......... .......... 71% 41.0M 2s 2024-09-17T23:48:13.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121000K .......... .......... .......... .......... .......... 71% 22.0M 2s 2024-09-17T23:48:13.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121050K .......... .......... .......... .......... .......... 71% 28.4M 2s 2024-09-17T23:48:13.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121100K .......... .......... .......... .......... .......... 71% 23.5M 2s 2024-09-17T23:48:13.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121150K .......... .......... .......... .......... .......... 71% 32.4M 2s 2024-09-17T23:48:13.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121200K .......... .......... .......... .......... .......... 71% 17.4M 2s 2024-09-17T23:48:13.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121250K .......... .......... .......... .......... .......... 71% 29.4M 2s 2024-09-17T23:48:13.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121300K .......... .......... .......... .......... .......... 71% 19.8M 2s 2024-09-17T23:48:13.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121350K .......... .......... .......... .......... .......... 72% 38.1M 2s 2024-09-17T23:48:13.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121400K .......... .......... .......... .......... .......... 72% 24.4M 2s 2024-09-17T23:48:13.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121450K .......... .......... .......... .......... .......... 72% 44.9M 2s 2024-09-17T23:48:13.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121500K .......... .......... .......... .......... .......... 72% 23.7M 2s 2024-09-17T23:48:13.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121550K .......... .......... .......... .......... .......... 72% 23.7M 2s 2024-09-17T23:48:13.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121600K .......... .......... .......... .......... .......... 72% 37.2M 2s 2024-09-17T23:48:13.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121650K .......... .......... .......... .......... .......... 72% 22.1M 2s 2024-09-17T23:48:13.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121700K .......... .......... .......... .......... .......... 72% 24.8M 2s 2024-09-17T23:48:13.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121750K .......... .......... .......... .......... .......... 72% 46.1M 2s 2024-09-17T23:48:13.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121800K .......... .......... .......... .......... .......... 72% 25.6M 2s 2024-09-17T23:48:13.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121850K .......... .......... .......... .......... .......... 72% 35.3M 2s 2024-09-17T23:48:13.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121900K .......... .......... .......... .......... .......... 72% 26.3M 2s 2024-09-17T23:48:13.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:121950K .......... .......... .......... .......... .......... 72% 23.5M 2s 2024-09-17T23:48:13.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122000K .......... .......... .......... .......... .......... 72% 45.3M 2s 2024-09-17T23:48:13.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122050K .......... .......... .......... .......... .......... 72% 25.0M 2s 2024-09-17T23:48:13.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122100K .......... .......... .......... .......... .......... 72% 37.8M 2s 2024-09-17T23:48:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122150K .......... .......... .......... .......... .......... 72% 24.3M 2s 2024-09-17T23:48:13.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122200K .......... .......... .......... .......... .......... 72% 32.5M 2s 2024-09-17T23:48:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122250K .......... .......... .......... .......... .......... 72% 38.5M 2s 2024-09-17T23:48:13.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122300K .......... .......... .......... .......... .......... 72% 22.9M 2s 2024-09-17T23:48:13.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122350K .......... .......... .......... .......... .......... 72% 22.8M 2s 2024-09-17T23:48:13.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122400K .......... .......... .......... .......... .......... 72% 35.8M 2s 2024-09-17T23:48:13.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122450K .......... .......... .......... .......... .......... 72% 25.2M 2s 2024-09-17T23:48:13.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122500K .......... .......... .......... .......... .......... 72% 44.7M 2s 2024-09-17T23:48:13.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122550K .......... .......... .......... .......... .......... 72% 23.8M 2s 2024-09-17T23:48:13.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122600K .......... .......... .......... .......... .......... 72% 32.1M 2s 2024-09-17T23:48:13.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122650K .......... .......... .......... .......... .......... 72% 26.0M 2s 2024-09-17T23:48:13.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122700K .......... .......... .......... .......... .......... 72% 43.5M 2s 2024-09-17T23:48:13.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122750K .......... .......... .......... .......... .......... 72% 16.0M 2s 2024-09-17T23:48:13.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122800K .......... .......... .......... .......... .......... 72% 52.2M 2s 2024-09-17T23:48:13.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122850K .......... .......... .......... .......... .......... 72% 24.1M 2s 2024-09-17T23:48:13.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122900K .......... .......... .......... .......... .......... 72% 37.3M 2s 2024-09-17T23:48:13.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:122950K .......... .......... .......... .......... .......... 72% 24.5M 2s 2024-09-17T23:48:13.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123000K .......... .......... .......... .......... .......... 72% 45.3M 2s 2024-09-17T23:48:13.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123050K .......... .......... .......... .......... .......... 73% 24.3M 2s 2024-09-17T23:48:13.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123100K .......... .......... .......... .......... .......... 73% 40.7M 2s 2024-09-17T23:48:13.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123150K .......... .......... .......... .......... .......... 73% 22.3M 2s 2024-09-17T23:48:13.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123200K .......... .......... .......... .......... .......... 73% 21.9M 2s 2024-09-17T23:48:13.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123250K .......... .......... .......... .......... .......... 73% 33.7M 2s 2024-09-17T23:48:13.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123300K .......... .......... .......... .......... .......... 73% 20.8M 2s 2024-09-17T23:48:13.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123350K .......... .......... .......... .......... .......... 73% 34.8M 2s 2024-09-17T23:48:13.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123400K .......... .......... .......... .......... .......... 73% 25.0M 2s 2024-09-17T23:48:13.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123450K .......... .......... .......... .......... .......... 73% 25.2M 2s 2024-09-17T23:48:13.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123500K .......... .......... .......... .......... .......... 73% 42.4M 2s 2024-09-17T23:48:13.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123550K .......... .......... .......... .......... .......... 73% 24.0M 2s 2024-09-17T23:48:13.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123600K .......... .......... .......... .......... .......... 73% 25.2M 2s 2024-09-17T23:48:13.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123650K .......... .......... .......... .......... .......... 73% 25.1M 2s 2024-09-17T23:48:13.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123700K .......... .......... .......... .......... .......... 73% 39.3M 2s 2024-09-17T23:48:13.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123750K .......... .......... .......... .......... .......... 73% 41.0M 2s 2024-09-17T23:48:13.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123800K .......... .......... .......... .......... .......... 73% 25.7M 2s 2024-09-17T23:48:13.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123850K .......... .......... .......... .......... .......... 73% 37.1M 2s 2024-09-17T23:48:13.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123900K .......... .......... .......... .......... .......... 73% 22.1M 2s 2024-09-17T23:48:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:123950K .......... .......... .......... .......... .......... 73% 20.2M 2s 2024-09-17T23:48:13.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124000K .......... .......... .......... .......... .......... 73% 26.0M 2s 2024-09-17T23:48:13.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124050K .......... .......... .......... .......... .......... 73% 40.0M 2s 2024-09-17T23:48:13.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124100K .......... .......... .......... .......... .......... 73% 22.9M 2s 2024-09-17T23:48:13.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124150K .......... .......... .......... .......... .......... 73% 34.6M 2s 2024-09-17T23:48:13.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124200K .......... .......... .......... .......... .......... 73% 23.9M 2s 2024-09-17T23:48:13.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124250K .......... .......... .......... .......... .......... 73% 43.3M 2s 2024-09-17T23:48:13.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124300K .......... .......... .......... .......... .......... 73% 22.1M 2s 2024-09-17T23:48:13.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124350K .......... .......... .......... .......... .......... 73% 21.6M 2s 2024-09-17T23:48:13.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124400K .......... .......... .......... .......... .......... 73% 38.2M 2s 2024-09-17T23:48:13.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124450K .......... .......... .......... .......... .......... 73% 21.3M 2s 2024-09-17T23:48:13.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124500K .......... .......... .......... .......... .......... 73% 22.9M 2s 2024-09-17T23:48:13.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124550K .......... .......... .......... .......... .......... 73% 40.3M 2s 2024-09-17T23:48:13.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124600K .......... .......... .......... .......... .......... 73% 24.8M 2s 2024-09-17T23:48:13.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124650K .......... .......... .......... .......... .......... 73% 38.7M 2s 2024-09-17T23:48:13.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124700K .......... .......... .......... .......... .......... 74% 22.8M 2s 2024-09-17T23:48:13.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124750K .......... .......... .......... .......... .......... 74% 18.4M 2s 2024-09-17T23:48:13.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124800K .......... .......... .......... .......... .......... 74% 41.6M 2s 2024-09-17T23:48:13.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124850K .......... .......... .......... .......... .......... 74% 24.1M 2s 2024-09-17T23:48:13.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124900K .......... .......... .......... .......... .......... 74% 40.1M 2s 2024-09-17T23:48:13.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:124950K .......... .......... .......... .......... .......... 74% 21.8M 2s 2024-09-17T23:48:13.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125000K .......... .......... .......... .......... .......... 74% 45.9M 2s 2024-09-17T23:48:13.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125050K .......... .......... .......... .......... .......... 74% 20.7M 2s 2024-09-17T23:48:13.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125100K .......... .......... .......... .......... .......... 74% 23.7M 2s 2024-09-17T23:48:13.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125150K .......... .......... .......... .......... .......... 74% 22.0M 2s 2024-09-17T23:48:13.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125200K .......... .......... .......... .......... .......... 74% 38.9M 2s 2024-09-17T23:48:13.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125250K .......... .......... .......... .......... .......... 74% 24.6M 2s 2024-09-17T23:48:13.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125300K .......... .......... .......... .......... .......... 74% 46.8M 2s 2024-09-17T23:48:13.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125350K .......... .......... .......... .......... .......... 74% 19.0M 2s 2024-09-17T23:48:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125400K .......... .......... .......... .......... .......... 74% 32.5M 2s 2024-09-17T23:48:13.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125450K .......... .......... .......... .......... .......... 74% 26.0M 2s 2024-09-17T23:48:13.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125500K .......... .......... .......... .......... .......... 74% 42.7M 2s 2024-09-17T23:48:13.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125550K .......... .......... .......... .......... .......... 74% 21.4M 2s 2024-09-17T23:48:13.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125600K .......... .......... .......... .......... .......... 74% 22.2M 2s 2024-09-17T23:48:13.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125650K .......... .......... .......... .......... .......... 74% 19.4M 2s 2024-09-17T23:48:13.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125700K .......... .......... .......... .......... .......... 74% 25.5M 2s 2024-09-17T23:48:13.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125750K .......... .......... .......... .......... .......... 74% 623K 2s 2024-09-17T23:48:13.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125800K .......... .......... .......... .......... .......... 74% 122M 2s 2024-09-17T23:48:13.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125850K .......... .......... .......... .......... .......... 74% 141M 2s 2024-09-17T23:48:13.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125900K .......... .......... .......... .......... .......... 74% 147M 2s 2024-09-17T23:48:13.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:125950K .......... .......... .......... .......... .......... 74% 101M 2s 2024-09-17T23:48:13.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126000K .......... .......... .......... .......... .......... 74% 114M 2s 2024-09-17T23:48:13.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126050K .......... .......... .......... .......... .......... 74% 144M 2s 2024-09-17T23:48:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126100K .......... .......... .......... .......... .......... 74% 124M 2s 2024-09-17T23:48:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126150K .......... .......... .......... .......... .......... 74% 130M 2s 2024-09-17T23:48:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126200K .......... .......... .......... .......... .......... 74% 145M 2s 2024-09-17T23:48:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126250K .......... .......... .......... .......... .......... 74% 144M 2s 2024-09-17T23:48:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126300K .......... .......... .......... .......... .......... 74% 146M 2s 2024-09-17T23:48:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126350K .......... .......... .......... .......... .......... 74% 124M 1s 2024-09-17T23:48:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126400K .......... .......... .......... .......... .......... 75% 143M 1s 2024-09-17T23:48:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126450K .......... .......... .......... .......... .......... 75% 146M 1s 2024-09-17T23:48:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126500K .......... .......... .......... .......... .......... 75% 140M 1s 2024-09-17T23:48:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126550K .......... .......... .......... .......... .......... 75% 135M 1s 2024-09-17T23:48:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126600K .......... .......... .......... .......... .......... 75% 147M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126650K .......... .......... .......... .......... .......... 75% 142M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126700K .......... .......... .......... .......... .......... 75% 141M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126750K .......... .......... .......... .......... .......... 75% 121M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126800K .......... .......... .......... .......... .......... 75% 149M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126850K .......... .......... .......... .......... .......... 75% 150M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126900K .......... .......... .......... .......... .......... 75% 147M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:126950K .......... .......... .......... .......... .......... 75% 134M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127000K .......... .......... .......... .......... .......... 75% 143M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127050K .......... .......... .......... .......... .......... 75% 147M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127100K .......... .......... .......... .......... .......... 75% 105M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127150K .......... .......... .......... .......... .......... 75% 120M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127200K .......... .......... .......... .......... .......... 75% 146M 1s 2024-09-17T23:48:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127250K .......... .......... .......... .......... .......... 75% 145M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127300K .......... .......... .......... .......... .......... 75% 141M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127350K .......... .......... .......... .......... .......... 75% 134M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127400K .......... .......... .......... .......... .......... 75% 146M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127450K .......... .......... .......... .......... .......... 75% 151M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127500K .......... .......... .......... .......... .......... 75% 139M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127550K .......... .......... .......... .......... .......... 75% 122M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127600K .......... .......... .......... .......... .......... 75% 143M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127650K .......... .......... .......... .......... .......... 75% 139M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127700K .......... .......... .......... .......... .......... 75% 145M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127750K .......... .......... .......... .......... .......... 75% 134M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127800K .......... .......... .......... .......... .......... 75% 150M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127850K .......... .......... .......... .......... .......... 75% 149M 1s 2024-09-17T23:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127900K .......... .......... .......... .......... .......... 75% 144M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:127950K .......... .......... .......... .......... .......... 75% 119M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128000K .......... .......... .......... .......... .......... 75% 148M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128050K .......... .......... .......... .......... .......... 75% 137M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128100K .......... .......... .......... .......... .......... 76% 138M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128150K .......... .......... .......... .......... .......... 76% 131M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128200K .......... .......... .......... .......... .......... 76% 128M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128250K .......... .......... .......... .......... .......... 76% 131M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128300K .......... .......... .......... .......... .......... 76% 147M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128350K .......... .......... .......... .......... .......... 76% 115M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128400K .......... .......... .......... .......... .......... 76% 136M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128450K .......... .......... .......... .......... .......... 76% 145M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128500K .......... .......... .......... .......... .......... 76% 140M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128550K .......... .......... .......... .......... .......... 76% 135M 1s 2024-09-17T23:48:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128600K .......... .......... .......... .......... .......... 76% 102M 1s 2024-09-17T23:48:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128650K .......... .......... .......... .......... .......... 76% 44.1M 1s 2024-09-17T23:48:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128700K .......... .......... .......... .......... .......... 76% 19.0M 1s 2024-09-17T23:48:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128750K .......... .......... .......... .......... .......... 76% 17.5M 1s 2024-09-17T23:48:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128800K .......... .......... .......... .......... .......... 76% 10.8M 1s 2024-09-17T23:48:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128850K .......... .......... .......... .......... .......... 76% 40.7M 1s 2024-09-17T23:48:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128900K .......... .......... .......... .......... .......... 76% 23.9M 1s 2024-09-17T23:48:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:128950K .......... .......... .......... .......... .......... 76% 28.6M 1s 2024-09-17T23:48:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129000K .......... .......... .......... .......... .......... 76% 36.1M 1s 2024-09-17T23:48:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129050K .......... .......... .......... .......... .......... 76% 23.8M 1s 2024-09-17T23:48:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129100K .......... .......... .......... .......... .......... 76% 38.4M 1s 2024-09-17T23:48:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129150K .......... .......... .......... .......... .......... 76% 23.5M 1s 2024-09-17T23:48:13.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129200K .......... .......... .......... .......... .......... 76% 28.9M 1s 2024-09-17T23:48:13.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129250K .......... .......... .......... .......... .......... 76% 25.2M 1s 2024-09-17T23:48:13.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129300K .......... .......... .......... .......... .......... 76% 37.0M 1s 2024-09-17T23:48:13.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129350K .......... .......... .......... .......... .......... 76% 37.6M 1s 2024-09-17T23:48:13.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129400K .......... .......... .......... .......... .......... 76% 22.0M 1s 2024-09-17T23:48:13.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129450K .......... .......... .......... .......... .......... 76% 33.1M 1s 2024-09-17T23:48:13.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129500K .......... .......... .......... .......... .......... 76% 32.8M 1s 2024-09-17T23:48:13.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129550K .......... .......... .......... .......... .......... 76% 23.6M 1s 2024-09-17T23:48:13.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129600K .......... .......... .......... .......... .......... 76% 25.1M 1s 2024-09-17T23:48:13.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129650K .......... .......... .......... .......... .......... 76% 40.3M 1s 2024-09-17T23:48:13.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129700K .......... .......... .......... .......... .......... 76% 28.9M 1s 2024-09-17T23:48:13.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129750K .......... .......... .......... .......... .......... 77% 21.4M 1s 2024-09-17T23:48:13.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129800K .......... .......... .......... .......... .......... 77% 36.8M 1s 2024-09-17T23:48:13.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129850K .......... .......... .......... .......... .......... 77% 40.5M 1s 2024-09-17T23:48:13.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129900K .......... .......... .......... .......... .......... 77% 26.8M 1s 2024-09-17T23:48:13.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:129950K .......... .......... .......... .......... .......... 77% 20.2M 1s 2024-09-17T23:48:13.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130000K .......... .......... .......... .......... .......... 77% 25.3M 1s 2024-09-17T23:48:13.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130050K .......... .......... .......... .......... .......... 77% 41.9M 1s 2024-09-17T23:48:13.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130100K .......... .......... .......... .......... .......... 77% 23.8M 1s 2024-09-17T23:48:13.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130150K .......... .......... .......... .......... .......... 77% 43.0M 1s 2024-09-17T23:48:13.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130200K .......... .......... .......... .......... .......... 77% 26.9M 1s 2024-09-17T23:48:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130250K .......... .......... .......... .......... .......... 77% 36.3M 1s 2024-09-17T23:48:13.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130300K .......... .......... .......... .......... .......... 77% 24.1M 1s 2024-09-17T23:48:13.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130350K .......... .......... .......... .......... .......... 77% 22.5M 1s 2024-09-17T23:48:13.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130400K .......... .......... .......... .......... .......... 77% 42.3M 1s 2024-09-17T23:48:13.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130450K .......... .......... .......... .......... .......... 77% 22.2M 1s 2024-09-17T23:48:13.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130500K .......... .......... .......... .......... .......... 77% 29.2M 1s 2024-09-17T23:48:13.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130550K .......... .......... .......... .......... .......... 77% 23.4M 1s 2024-09-17T23:48:13.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130600K .......... .......... .......... .......... .......... 77% 125M 1s 2024-09-17T23:48:13.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130650K .......... .......... .......... .......... .......... 77% 22.5M 1s 2024-09-17T23:48:13.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130700K .......... .......... .......... .......... .......... 77% 28.4M 1s 2024-09-17T23:48:13.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130750K .......... .......... .......... .......... .......... 77% 24.3M 1s 2024-09-17T23:48:13.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130800K .......... .......... .......... .......... .......... 77% 26.0M 1s 2024-09-17T23:48:13.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130850K .......... .......... .......... .......... .......... 77% 25.3M 1s 2024-09-17T23:48:13.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130900K .......... .......... .......... .......... .......... 77% 62.8M 1s 2024-09-17T23:48:13.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:130950K .......... .......... .......... .......... .......... 77% 22.5M 1s 2024-09-17T23:48:13.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131000K .......... .......... .......... .......... .......... 77% 36.8M 1s 2024-09-17T23:48:13.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131050K .......... .......... .......... .......... .......... 77% 21.8M 1s 2024-09-17T23:48:13.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131100K .......... .......... .......... .......... .......... 77% 48.8M 1s 2024-09-17T23:48:13.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131150K .......... .......... .......... .......... .......... 77% 19.5M 1s 2024-09-17T23:48:13.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131200K .......... .......... .......... .......... .......... 77% 21.0M 1s 2024-09-17T23:48:13.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131250K .......... .......... .......... .......... .......... 77% 23.1M 1s 2024-09-17T23:48:13.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131300K .......... .......... .......... .......... .......... 77% 37.3M 1s 2024-09-17T23:48:13.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131350K .......... .......... .......... .......... .......... 77% 25.3M 1s 2024-09-17T23:48:13.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131400K .......... .......... .......... .......... .......... 77% 41.5M 1s 2024-09-17T23:48:13.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131450K .......... .......... .......... .......... .......... 78% 25.4M 1s 2024-09-17T23:48:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131500K .......... .......... .......... .......... .......... 78% 41.8M 1s 2024-09-17T23:48:13.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131550K .......... .......... .......... .......... .......... 78% 24.4M 1s 2024-09-17T23:48:13.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131600K .......... .......... .......... .......... .......... 78% 23.2M 1s 2024-09-17T23:48:13.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131650K .......... .......... .......... .......... .......... 78% 44.0M 1s 2024-09-17T23:48:13.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131700K .......... .......... .......... .......... .......... 78% 22.7M 1s 2024-09-17T23:48:13.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131750K .......... .......... .......... .......... .......... 78% 21.9M 1s 2024-09-17T23:48:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131800K .......... .......... .......... .......... .......... 78% 32.3M 1s 2024-09-17T23:48:13.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131850K .......... .......... .......... .......... .......... 78% 22.2M 1s 2024-09-17T23:48:13.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131900K .......... .......... .......... .......... .......... 78% 44.9M 1s 2024-09-17T23:48:13.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:131950K .......... .......... .......... .......... .......... 78% 22.1M 1s 2024-09-17T23:48:13.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132000K .......... .......... .......... .......... .......... 78% 16.2M 1s 2024-09-17T23:48:13.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132050K .......... .......... .......... .......... .......... 78% 35.6M 1s 2024-09-17T23:48:13.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132100K .......... .......... .......... .......... .......... 78% 25.9M 1s 2024-09-17T23:48:13.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132150K .......... .......... .......... .......... .......... 78% 43.9M 1s 2024-09-17T23:48:13.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132200K .......... .......... .......... .......... .......... 78% 29.3M 1s 2024-09-17T23:48:13.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132250K .......... .......... .......... .......... .......... 78% 25.2M 1s 2024-09-17T23:48:13.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132300K .......... .......... .......... .......... .......... 78% 28.3M 1s 2024-09-17T23:48:13.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132350K .......... .......... .......... .......... .......... 78% 16.2M 1s 2024-09-17T23:48:13.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132400K .......... .......... .......... .......... .......... 78% 59.5M 1s 2024-09-17T23:48:13.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132450K .......... .......... .......... .......... .......... 78% 23.0M 1s 2024-09-17T23:48:13.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132500K .......... .......... .......... .......... .......... 78% 13.4M 1s 2024-09-17T23:48:13.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132550K .......... .......... .......... .......... .......... 78% 16.8M 1s 2024-09-17T23:48:13.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132600K .......... .......... .......... .......... .......... 78% 21.3M 1s 2024-09-17T23:48:13.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132650K .......... .......... .......... .......... .......... 78% 46.1M 1s 2024-09-17T23:48:13.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132700K .......... .......... .......... .......... .......... 78% 25.5M 1s 2024-09-17T23:48:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132750K .......... .......... .......... .......... .......... 78% 23.0M 1s 2024-09-17T23:48:13.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132800K .......... .......... .......... .......... .......... 78% 39.4M 1s 2024-09-17T23:48:13.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132850K .......... .......... .......... .......... .......... 78% 25.5M 1s 2024-09-17T23:48:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132900K .......... .......... .......... .......... .......... 78% 21.6M 1s 2024-09-17T23:48:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:132950K .......... .......... .......... .......... .......... 78% 48.7M 1s 2024-09-17T23:48:13.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133000K .......... .......... .......... .......... .......... 78% 35.3M 1s 2024-09-17T23:48:13.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133050K .......... .......... .......... .......... .......... 78% 24.8M 1s 2024-09-17T23:48:13.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133100K .......... .......... .......... .......... .......... 78% 19.3M 1s 2024-09-17T23:48:13.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133150K .......... .......... .......... .......... .......... 79% 34.0M 1s 2024-09-17T23:48:13.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133200K .......... .......... .......... .......... .......... 79% 42.4M 1s 2024-09-17T23:48:13.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133250K .......... .......... .......... .......... .......... 79% 25.6M 1s 2024-09-17T23:48:13.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133300K .......... .......... .......... .......... .......... 79% 39.3M 1s 2024-09-17T23:48:13.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133350K .......... .......... .......... .......... .......... 79% 23.0M 1s 2024-09-17T23:48:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133400K .......... .......... .......... .......... .......... 79% 48.5M 1s 2024-09-17T23:48:13.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133450K .......... .......... .......... .......... .......... 79% 23.4M 1s 2024-09-17T23:48:13.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133500K .......... .......... .......... .......... .......... 79% 27.0M 1s 2024-09-17T23:48:13.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133550K .......... .......... .......... .......... .......... 79% 26.0M 1s 2024-09-17T23:48:13.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133600K .......... .......... .......... .......... .......... 79% 28.1M 1s 2024-09-17T23:48:13.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133650K .......... .......... .......... .......... .......... 79% 25.2M 1s 2024-09-17T23:48:13.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133700K .......... .......... .......... .......... .......... 79% 43.9M 1s 2024-09-17T23:48:13.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133750K .......... .......... .......... .......... .......... 79% 24.8M 1s 2024-09-17T23:48:13.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133800K .......... .......... .......... .......... .......... 79% 40.4M 1s 2024-09-17T23:48:13.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133850K .......... .......... .......... .......... .......... 79% 24.5M 1s 2024-09-17T23:48:13.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133900K .......... .......... .......... .......... .......... 79% 42.8M 1s 2024-09-17T23:48:13.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:133950K .......... .......... .......... .......... .......... 79% 23.7M 1s 2024-09-17T23:48:13.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134000K .......... .......... .......... .......... .......... 79% 30.3M 1s 2024-09-17T23:48:13.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134050K .......... .......... .......... .......... .......... 79% 19.8M 1s 2024-09-17T23:48:13.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134100K .......... .......... .......... .......... .......... 79% 28.6M 1s 2024-09-17T23:48:13.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134150K .......... .......... .......... .......... .......... 79% 27.7M 1s 2024-09-17T23:48:13.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134200K .......... .......... .......... .......... .......... 79% 42.9M 1s 2024-09-17T23:48:13.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134250K .......... .......... .......... .......... .......... 79% 26.1M 1s 2024-09-17T23:48:13.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134300K .......... .......... .......... .......... .......... 79% 39.5M 1s 2024-09-17T23:48:13.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134350K .......... .......... .......... .......... .......... 79% 24.2M 1s 2024-09-17T23:48:13.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134400K .......... .......... .......... .......... .......... 79% 24.8M 1s 2024-09-17T23:48:13.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134450K .......... .......... .......... .......... .......... 79% 45.8M 1s 2024-09-17T23:48:13.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134500K .......... .......... .......... .......... .......... 79% 23.5M 1s 2024-09-17T23:48:13.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134550K .......... .......... .......... .......... .......... 79% 39.2M 1s 2024-09-17T23:48:13.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134600K .......... .......... .......... .......... .......... 79% 22.6M 1s 2024-09-17T23:48:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134650K .......... .......... .......... .......... .......... 79% 23.4M 1s 2024-09-17T23:48:13.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134700K .......... .......... .......... .......... .......... 79% 33.1M 1s 2024-09-17T23:48:13.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134750K .......... .......... .......... .......... .......... 79% 20.1M 1s 2024-09-17T23:48:13.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134800K .......... .......... .......... .......... .......... 79% 21.7M 1s 2024-09-17T23:48:13.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134850K .......... .......... .......... .......... .......... 80% 40.4M 1s 2024-09-17T23:48:13.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134900K .......... .......... .......... .......... .......... 80% 25.1M 1s 2024-09-17T23:48:13.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:134950K .......... .......... .......... .......... .......... 80% 38.1M 1s 2024-09-17T23:48:13.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135000K .......... .......... .......... .......... .......... 80% 27.5M 1s 2024-09-17T23:48:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135050K .......... .......... .......... .......... .......... 80% 29.6M 1s 2024-09-17T23:48:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135100K .......... .......... .......... .......... .......... 80% 27.8M 1s 2024-09-17T23:48:13.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135150K .......... .......... .......... .......... .......... 80% 22.8M 1s 2024-09-17T23:48:13.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135200K .......... .......... .......... .......... .......... 80% 25.0M 1s 2024-09-17T23:48:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135250K .......... .......... .......... .......... .......... 80% 41.2M 1s 2024-09-17T23:48:13.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135300K .......... .......... .......... .......... .......... 80% 22.8M 1s 2024-09-17T23:48:13.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135350K .......... .......... .......... .......... .......... 80% 37.9M 1s 2024-09-17T23:48:13.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135400K .......... .......... .......... .......... .......... 80% 28.1M 1s 2024-09-17T23:48:13.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135450K .......... .......... .......... .......... .......... 80% 31.0M 1s 2024-09-17T23:48:13.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135500K .......... .......... .......... .......... .......... 80% 20.3M 1s 2024-09-17T23:48:13.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135550K .......... .......... .......... .......... .......... 80% 22.2M 1s 2024-09-17T23:48:13.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135600K .......... .......... .......... .......... .......... 80% 37.0M 1s 2024-09-17T23:48:13.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135650K .......... .......... .......... .......... .......... 80% 17.6M 1s 2024-09-17T23:48:13.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135700K .......... .......... .......... .......... .......... 80% 37.8M 1s 2024-09-17T23:48:13.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135750K .......... .......... .......... .......... .......... 80% 24.3M 1s 2024-09-17T23:48:13.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135800K .......... .......... .......... .......... .......... 80% 34.0M 1s 2024-09-17T23:48:13.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135850K .......... .......... .......... .......... .......... 80% 27.7M 1s 2024-09-17T23:48:13.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135900K .......... .......... .......... .......... .......... 80% 23.1M 1s 2024-09-17T23:48:13.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:135950K .......... .......... .......... .......... .......... 80% 29.1M 1s 2024-09-17T23:48:13.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136000K .......... .......... .......... .......... .......... 80% 35.9M 1s 2024-09-17T23:48:13.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136050K .......... .......... .......... .......... .......... 80% 18.0M 1s 2024-09-17T23:48:13.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136100K .......... .......... .......... .......... .......... 80% 31.7M 1s 2024-09-17T23:48:13.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136150K .......... .......... .......... .......... .......... 80% 20.0M 1s 2024-09-17T23:48:13.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136200K .......... .......... .......... .......... .......... 80% 29.3M 1s 2024-09-17T23:48:13.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136250K .......... .......... .......... .......... .......... 80% 17.7M 1s 2024-09-17T23:48:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136300K .......... .......... .......... .......... .......... 80% 18.9M 1s 2024-09-17T23:48:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136350K .......... .......... .......... .......... .......... 80% 17.9M 1s 2024-09-17T23:48:13.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136400K .......... .......... .......... .......... .......... 80% 30.8M 1s 2024-09-17T23:48:13.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136450K .......... .......... .......... .......... .......... 80% 21.7M 1s 2024-09-17T23:48:13.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136500K .......... .......... .......... .......... .......... 81% 51.5M 1s 2024-09-17T23:48:13.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136550K .......... .......... .......... .......... .......... 81% 23.6M 1s 2024-09-17T23:48:13.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136600K .......... .......... .......... .......... .......... 81% 33.5M 1s 2024-09-17T23:48:13.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136650K .......... .......... .......... .......... .......... 81% 24.8M 1s 2024-09-17T23:48:13.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136700K .......... .......... .......... .......... .......... 81% 45.6M 1s 2024-09-17T23:48:13.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136750K .......... .......... .......... .......... .......... 81% 23.8M 1s 2024-09-17T23:48:13.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136800K .......... .......... .......... .......... .......... 81% 25.3M 1s 2024-09-17T23:48:13.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136850K .......... .......... .......... .......... .......... 81% 37.7M 1s 2024-09-17T23:48:13.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136900K .......... .......... .......... .......... .......... 81% 23.5M 1s 2024-09-17T23:48:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:136950K .......... .......... .......... .......... .......... 81% 23.5M 1s 2024-09-17T23:48:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137000K .......... .......... .......... .......... .......... 81% 47.9M 1s 2024-09-17T23:48:13.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137050K .......... .......... .......... .......... .......... 81% 24.2M 1s 2024-09-17T23:48:13.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137100K .......... .......... .......... .......... .......... 81% 38.5M 1s 2024-09-17T23:48:13.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137150K .......... .......... .......... .......... .......... 81% 22.5M 1s 2024-09-17T23:48:13.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137200K .......... .......... .......... .......... .......... 81% 25.5M 1s 2024-09-17T23:48:13.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137250K .......... .......... .......... .......... .......... 81% 44.8M 1s 2024-09-17T23:48:13.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137300K .......... .......... .......... .......... .......... 81% 25.4M 1s 2024-09-17T23:48:13.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137350K .......... .......... .......... .......... .......... 81% 36.2M 1s 2024-09-17T23:48:13.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137400K .......... .......... .......... .......... .......... 81% 24.9M 1s 2024-09-17T23:48:13.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137450K .......... .......... .......... .......... .......... 81% 23.5M 1s 2024-09-17T23:48:13.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137500K .......... .......... .......... .......... .......... 81% 44.5M 1s 2024-09-17T23:48:13.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137550K .......... .......... .......... .......... .......... 81% 24.2M 1s 2024-09-17T23:48:13.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137600K .......... .......... .......... .......... .......... 81% 24.4M 1s 2024-09-17T23:48:13.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137650K .......... .......... .......... .......... .......... 81% 39.8M 1s 2024-09-17T23:48:13.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137700K .......... .......... .......... .......... .......... 81% 23.7M 1s 2024-09-17T23:48:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137750K .......... .......... .......... .......... .......... 81% 39.8M 1s 2024-09-17T23:48:13.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137800K .......... .......... .......... .......... .......... 81% 24.2M 1s 2024-09-17T23:48:13.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137850K .......... .......... .......... .......... .......... 81% 41.4M 1s 2024-09-17T23:48:13.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137900K .......... .......... .......... .......... .......... 81% 25.3M 1s 2024-09-17T23:48:13.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:137950K .......... .......... .......... .......... .......... 81% 24.2M 1s 2024-09-17T23:48:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138000K .......... .......... .......... .......... .......... 81% 23.3M 1s 2024-09-17T23:48:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138050K .......... .......... .......... .......... .......... 81% 39.6M 1s 2024-09-17T23:48:13.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138100K .......... .......... .......... .......... .......... 81% 25.0M 1s 2024-09-17T23:48:13.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138150K .......... .......... .......... .......... .......... 81% 39.1M 1s 2024-09-17T23:48:13.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138200K .......... .......... .......... .......... .......... 82% 21.2M 1s 2024-09-17T23:48:13.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138250K .......... .......... .......... .......... .......... 82% 30.9M 1s 2024-09-17T23:48:13.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138300K .......... .......... .......... .......... .......... 82% 24.6M 1s 2024-09-17T23:48:13.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138350K .......... .......... .......... .......... .......... 82% 23.4M 1s 2024-09-17T23:48:13.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138400K .......... .......... .......... .......... .......... 82% 21.4M 1s 2024-09-17T23:48:13.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138450K .......... .......... .......... .......... .......... 82% 41.3M 1s 2024-09-17T23:48:13.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138500K .......... .......... .......... .......... .......... 82% 41.6M 1s 2024-09-17T23:48:13.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138550K .......... .......... .......... .......... .......... 82% 24.7M 1s 2024-09-17T23:48:13.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138600K .......... .......... .......... .......... .......... 82% 23.5M 1s 2024-09-17T23:48:13.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138650K .......... .......... .......... .......... .......... 82% 43.2M 1s 2024-09-17T23:48:13.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138700K .......... .......... .......... .......... .......... 82% 24.0M 1s 2024-09-17T23:48:13.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138750K .......... .......... .......... .......... .......... 82% 23.6M 1s 2024-09-17T23:48:13.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138800K .......... .......... .......... .......... .......... 82% 43.4M 1s 2024-09-17T23:48:13.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138850K .......... .......... .......... .......... .......... 82% 23.2M 1s 2024-09-17T23:48:13.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138900K .......... .......... .......... .......... .......... 82% 38.2M 1s 2024-09-17T23:48:13.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:138950K .......... .......... .......... .......... .......... 82% 22.7M 1s 2024-09-17T23:48:13.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139000K .......... .......... .......... .......... .......... 82% 46.1M 1s 2024-09-17T23:48:13.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139050K .......... .......... .......... .......... .......... 82% 20.3M 1s 2024-09-17T23:48:13.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139100K .......... .......... .......... .......... .......... 82% 36.5M 1s 2024-09-17T23:48:13.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139150K .......... .......... .......... .......... .......... 82% 16.6M 1s 2024-09-17T23:48:13.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139200K .......... .......... .......... .......... .......... 82% 29.9M 1s 2024-09-17T23:48:14.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139250K .......... .......... .......... .......... .......... 82% 22.9M 1s 2024-09-17T23:48:14.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139300K .......... .......... .......... .......... .......... 82% 44.0M 1s 2024-09-17T23:48:14.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139350K .......... .......... .......... .......... .......... 82% 18.0M 1s 2024-09-17T23:48:14.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139400K .......... .......... .......... .......... .......... 82% 35.7M 1s 2024-09-17T23:48:14.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139450K .......... .......... .......... .......... .......... 82% 24.4M 1s 2024-09-17T23:48:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139500K .......... .......... .......... .......... .......... 82% 42.8M 1s 2024-09-17T23:48:14.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139550K .......... .......... .......... .......... .......... 82% 24.1M 1s 2024-09-17T23:48:14.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139600K .......... .......... .......... .......... .......... 82% 26.7M 1s 2024-09-17T23:48:14.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139650K .......... .......... .......... .......... .......... 82% 37.1M 1s 2024-09-17T23:48:14.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139700K .......... .......... .......... .......... .......... 82% 17.4M 1s 2024-09-17T23:48:14.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139750K .......... .......... .......... .......... .......... 82% 25.2M 1s 2024-09-17T23:48:14.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139800K .......... .......... .......... .......... .......... 82% 44.7M 1s 2024-09-17T23:48:14.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139850K .......... .......... .......... .......... .......... 82% 18.2M 1s 2024-09-17T23:48:14.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139900K .......... .......... .......... .......... .......... 83% 22.8M 1s 2024-09-17T23:48:14.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:139950K .......... .......... .......... .......... .......... 83% 20.2M 1s 2024-09-17T23:48:14.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140000K .......... .......... .......... .......... .......... 83% 13.6M 1s 2024-09-17T23:48:14.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140050K .......... .......... .......... .......... .......... 83% 45.3M 1s 2024-09-17T23:48:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140100K .......... .......... .......... .......... .......... 83% 20.0M 1s 2024-09-17T23:48:14.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140150K .......... .......... .......... .......... .......... 83% 25.1M 1s 2024-09-17T23:48:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140200K .......... .......... .......... .......... .......... 83% 25.5M 1s 2024-09-17T23:48:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140250K .......... .......... .......... .......... .......... 83% 27.6M 1s 2024-09-17T23:48:14.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140300K .......... .......... .......... .......... .......... 83% 54.0M 1s 2024-09-17T23:48:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140350K .......... .......... .......... .......... .......... 83% 22.9M 1s 2024-09-17T23:48:14.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140400K .......... .......... .......... .......... .......... 83% 25.1M 1s 2024-09-17T23:48:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140450K .......... .......... .......... .......... .......... 83% 34.5M 1s 2024-09-17T23:48:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140500K .......... .......... .......... .......... .......... 83% 25.1M 1s 2024-09-17T23:48:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140550K .......... .......... .......... .......... .......... 83% 41.0M 1s 2024-09-17T23:48:14.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140600K .......... .......... .......... .......... .......... 83% 36.8M 1s 2024-09-17T23:48:14.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140650K .......... .......... .......... .......... .......... 83% 25.9M 1s 2024-09-17T23:48:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140700K .......... .......... .......... .......... .......... 83% 26.6M 1s 2024-09-17T23:48:14.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140750K .......... .......... .......... .......... .......... 83% 23.1M 1s 2024-09-17T23:48:14.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140800K .......... .......... .......... .......... .......... 83% 23.9M 1s 2024-09-17T23:48:14.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140850K .......... .......... .......... .......... .......... 83% 51.1M 1s 2024-09-17T23:48:14.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140900K .......... .......... .......... .......... .......... 83% 22.1M 1s 2024-09-17T23:48:14.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:140950K .......... .......... .......... .......... .......... 83% 38.4M 1s 2024-09-17T23:48:14.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141000K .......... .......... .......... .......... .......... 83% 25.2M 1s 2024-09-17T23:48:14.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141050K .......... .......... .......... .......... .......... 83% 41.3M 1s 2024-09-17T23:48:14.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141100K .......... .......... .......... .......... .......... 83% 26.3M 1s 2024-09-17T23:48:14.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141150K .......... .......... .......... .......... .......... 83% 22.8M 1s 2024-09-17T23:48:14.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141200K .......... .......... .......... .......... .......... 83% 40.6M 1s 2024-09-17T23:48:14.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141250K .......... .......... .......... .......... .......... 83% 37.2M 1s 2024-09-17T23:48:14.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141300K .......... .......... .......... .......... .......... 83% 26.4M 1s 2024-09-17T23:48:14.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141350K .......... .......... .......... .......... .......... 83% 21.1M 1s 2024-09-17T23:48:14.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141400K .......... .......... .......... .......... .......... 83% 53.4M 1s 2024-09-17T23:48:14.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141450K .......... .......... .......... .......... .......... 83% 23.3M 1s 2024-09-17T23:48:14.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141500K .......... .......... .......... .......... .......... 83% 23.3M 1s 2024-09-17T23:48:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141550K .......... .......... .......... .......... .......... 84% 23.8M 1s 2024-09-17T23:48:14.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141600K .......... .......... .......... .......... .......... 84% 48.0M 1s 2024-09-17T23:48:14.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141650K .......... .......... .......... .......... .......... 84% 23.2M 1s 2024-09-17T23:48:14.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141700K .......... .......... .......... .......... .......... 84% 41.4M 1s 2024-09-17T23:48:14.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141750K .......... .......... .......... .......... .......... 84% 22.0M 1s 2024-09-17T23:48:14.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141800K .......... .......... .......... .......... .......... 84% 39.3M 1s 2024-09-17T23:48:14.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141850K .......... .......... .......... .......... .......... 84% 20.3M 1s 2024-09-17T23:48:14.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141900K .......... .......... .......... .......... .......... 84% 23.2M 1s 2024-09-17T23:48:14.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:141950K .......... .......... .......... .......... .......... 84% 21.9M 1s 2024-09-17T23:48:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142000K .......... .......... .......... .......... .......... 84% 32.4M 1s 2024-09-17T23:48:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142050K .......... .......... .......... .......... .......... 84% 25.0M 1s 2024-09-17T23:48:14.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142100K .......... .......... .......... .......... .......... 84% 47.2M 1s 2024-09-17T23:48:14.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142150K .......... .......... .......... .......... .......... 84% 28.1M 1s 2024-09-17T23:48:14.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142200K .......... .......... .......... .......... .......... 84% 31.6M 1s 2024-09-17T23:48:14.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142250K .......... .......... .......... .......... .......... 84% 24.0M 1s 2024-09-17T23:48:14.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142300K .......... .......... .......... .......... .......... 84% 42.4M 1s 2024-09-17T23:48:14.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142350K .......... .......... .......... .......... .......... 84% 24.2M 1s 2024-09-17T23:48:14.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142400K .......... .......... .......... .......... .......... 84% 28.7M 1s 2024-09-17T23:48:14.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142450K .......... .......... .......... .......... .......... 84% 34.5M 1s 2024-09-17T23:48:14.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142500K .......... .......... .......... .......... .......... 84% 21.5M 1s 2024-09-17T23:48:14.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142550K .......... .......... .......... .......... .......... 84% 21.9M 1s 2024-09-17T23:48:14.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142600K .......... .......... .......... .......... .......... 84% 45.9M 1s 2024-09-17T23:48:14.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142650K .......... .......... .......... .......... .......... 84% 20.9M 1s 2024-09-17T23:48:14.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142700K .......... .......... .......... .......... .......... 84% 37.2M 1s 2024-09-17T23:48:14.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142750K .......... .......... .......... .......... .......... 84% 19.1M 1s 2024-09-17T23:48:14.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142800K .......... .......... .......... .......... .......... 84% 27.9M 1s 2024-09-17T23:48:14.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142850K .......... .......... .......... .......... .......... 84% 35.5M 1s 2024-09-17T23:48:14.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142900K .......... .......... .......... .......... .......... 84% 21.8M 1s 2024-09-17T23:48:14.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:142950K .......... .......... .......... .......... .......... 84% 36.1M 1s 2024-09-17T23:48:14.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143000K .......... .......... .......... .......... .......... 84% 22.0M 1s 2024-09-17T23:48:14.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143050K .......... .......... .......... .......... .......... 84% 33.5M 1s 2024-09-17T23:48:14.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143100K .......... .......... .......... .......... .......... 84% 26.9M 1s 2024-09-17T23:48:14.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143150K .......... .......... .......... .......... .......... 84% 17.6M 1s 2024-09-17T23:48:14.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143200K .......... .......... .......... .......... .......... 84% 28.4M 1s 2024-09-17T23:48:14.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143250K .......... .......... .......... .......... .......... 85% 36.7M 1s 2024-09-17T23:48:14.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143300K .......... .......... .......... .......... .......... 85% 26.7M 1s 2024-09-17T23:48:14.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143350K .......... .......... .......... .......... .......... 85% 33.8M 1s 2024-09-17T23:48:14.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143400K .......... .......... .......... .......... .......... 85% 21.5M 1s 2024-09-17T23:48:14.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143450K .......... .......... .......... .......... .......... 85% 33.6M 1s 2024-09-17T23:48:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143500K .......... .......... .......... .......... .......... 85% 20.9M 1s 2024-09-17T23:48:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143550K .......... .......... .......... .......... .......... 85% 20.9M 1s 2024-09-17T23:48:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143600K .......... .......... .......... .......... .......... 85% 24.8M 1s 2024-09-17T23:48:14.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143650K .......... .......... .......... .......... .......... 85% 20.2M 1s 2024-09-17T23:48:14.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143700K .......... .......... .......... .......... .......... 85% 33.8M 1s 2024-09-17T23:48:14.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143750K .......... .......... .......... .......... .......... 85% 16.8M 1s 2024-09-17T23:48:14.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143800K .......... .......... .......... .......... .......... 85% 18.1M 1s 2024-09-17T23:48:14.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143850K .......... .......... .......... .......... .......... 85% 48.1M 1s 2024-09-17T23:48:14.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143900K .......... .......... .......... .......... .......... 85% 17.8M 1s 2024-09-17T23:48:14.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:143950K .......... .......... .......... .......... .......... 85% 25.7M 1s 2024-09-17T23:48:14.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144000K .......... .......... .......... .......... .......... 85% 35.7M 1s 2024-09-17T23:48:14.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144050K .......... .......... .......... .......... .......... 85% 25.1M 1s 2024-09-17T23:48:14.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144100K .......... .......... .......... .......... .......... 85% 41.8M 1s 2024-09-17T23:48:14.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144150K .......... .......... .......... .......... .......... 85% 33.1M 1s 2024-09-17T23:48:14.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144200K .......... .......... .......... .......... .......... 85% 26.2M 1s 2024-09-17T23:48:14.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144250K .......... .......... .......... .......... .......... 85% 30.9M 1s 2024-09-17T23:48:14.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144300K .......... .......... .......... .......... .......... 85% 20.5M 1s 2024-09-17T23:48:14.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144350K .......... .......... .......... .......... .......... 85% 25.1M 1s 2024-09-17T23:48:14.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144400K .......... .......... .......... .......... .......... 85% 43.1M 1s 2024-09-17T23:48:14.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144450K .......... .......... .......... .......... .......... 85% 24.7M 1s 2024-09-17T23:48:14.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144500K .......... .......... .......... .......... .......... 85% 41.0M 1s 2024-09-17T23:48:14.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144550K .......... .......... .......... .......... .......... 85% 24.1M 1s 2024-09-17T23:48:14.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144600K .......... .......... .......... .......... .......... 85% 46.1M 1s 2024-09-17T23:48:14.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144650K .......... .......... .......... .......... .......... 85% 23.3M 1s 2024-09-17T23:48:14.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144700K .......... .......... .......... .......... .......... 85% 36.8M 1s 2024-09-17T23:48:14.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144750K .......... .......... .......... .......... .......... 85% 24.9M 1s 2024-09-17T23:48:14.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144800K .......... .......... .......... .......... .......... 85% 24.1M 1s 2024-09-17T23:48:14.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144850K .......... .......... .......... .......... .......... 85% 24.1M 1s 2024-09-17T23:48:14.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144900K .......... .......... .......... .......... .......... 85% 43.7M 1s 2024-09-17T23:48:14.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:144950K .......... .......... .......... .......... .......... 86% 24.3M 1s 2024-09-17T23:48:14.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145000K .......... .......... .......... .......... .......... 86% 31.4M 1s 2024-09-17T23:48:14.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145050K .......... .......... .......... .......... .......... 86% 30.5M 1s 2024-09-17T23:48:14.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145100K .......... .......... .......... .......... .......... 86% 43.3M 1s 2024-09-17T23:48:14.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145150K .......... .......... .......... .......... .......... 86% 21.5M 1s 2024-09-17T23:48:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145200K .......... .......... .......... .......... .......... 86% 27.8M 1s 2024-09-17T23:48:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145250K .......... .......... .......... .......... .......... 86% 35.0M 1s 2024-09-17T23:48:14.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145300K .......... .......... .......... .......... .......... 86% 24.5M 1s 2024-09-17T23:48:14.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145350K .......... .......... .......... .......... .......... 86% 20.5M 1s 2024-09-17T23:48:14.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145400K .......... .......... .......... .......... .......... 86% 37.3M 1s 2024-09-17T23:48:14.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145450K .......... .......... .......... .......... .......... 86% 20.5M 1s 2024-09-17T23:48:14.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145500K .......... .......... .......... .......... .......... 86% 40.4M 1s 2024-09-17T23:48:14.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145550K .......... .......... .......... .......... .......... 86% 22.6M 1s 2024-09-17T23:48:14.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145600K .......... .......... .......... .......... .......... 86% 21.6M 1s 2024-09-17T23:48:14.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145650K .......... .......... .......... .......... .......... 86% 41.9M 1s 2024-09-17T23:48:14.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145700K .......... .......... .......... .......... .......... 86% 27.1M 1s 2024-09-17T23:48:14.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145750K .......... .......... .......... .......... .......... 86% 38.6M 1s 2024-09-17T23:48:14.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145800K .......... .......... .......... .......... .......... 86% 24.2M 1s 2024-09-17T23:48:14.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145850K .......... .......... .......... .......... .......... 86% 41.7M 1s 2024-09-17T23:48:14.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145900K .......... .......... .......... .......... .......... 86% 23.3M 1s 2024-09-17T23:48:14.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:145950K .......... .......... .......... .......... .......... 86% 23.5M 1s 2024-09-17T23:48:14.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146000K .......... .......... .......... .......... .......... 86% 24.6M 1s 2024-09-17T23:48:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146050K .......... .......... .......... .......... .......... 86% 41.5M 1s 2024-09-17T23:48:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146100K .......... .......... .......... .......... .......... 86% 20.9M 1s 2024-09-17T23:48:14.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146150K .......... .......... .......... .......... .......... 86% 39.9M 1s 2024-09-17T23:48:14.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146200K .......... .......... .......... .......... .......... 86% 23.6M 1s 2024-09-17T23:48:14.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146250K .......... .......... .......... .......... .......... 86% 41.5M 1s 2024-09-17T23:48:14.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146300K .......... .......... .......... .......... .......... 86% 20.2M 1s 2024-09-17T23:48:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146350K .......... .......... .......... .......... .......... 86% 23.3M 1s 2024-09-17T23:48:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146400K .......... .......... .......... .......... .......... 86% 43.2M 1s 2024-09-17T23:48:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146450K .......... .......... .......... .......... .......... 86% 24.9M 1s 2024-09-17T23:48:14.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146500K .......... .......... .......... .......... .......... 86% 19.5M 1s 2024-09-17T23:48:14.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146550K .......... .......... .......... .......... .......... 86% 34.0M 1s 2024-09-17T23:48:14.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146600K .......... .......... .......... .......... .......... 86% 30.7M 1s 2024-09-17T23:48:14.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146650K .......... .......... .......... .......... .......... 87% 27.4M 1s 2024-09-17T23:48:14.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146700K .......... .......... .......... .......... .......... 87% 17.6M 1s 2024-09-17T23:48:14.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146750K .......... .......... .......... .......... .......... 87% 24.3M 1s 2024-09-17T23:48:14.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146800K .......... .......... .......... .......... .......... 87% 38.4M 1s 2024-09-17T23:48:14.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146850K .......... .......... .......... .......... .......... 87% 25.5M 1s 2024-09-17T23:48:14.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146900K .......... .......... .......... .......... .......... 87% 42.9M 1s 2024-09-17T23:48:14.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:146950K .......... .......... .......... .......... .......... 87% 24.0M 1s 2024-09-17T23:48:14.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147000K .......... .......... .......... .......... .......... 87% 39.3M 1s 2024-09-17T23:48:14.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147050K .......... .......... .......... .......... .......... 87% 18.8M 1s 2024-09-17T23:48:14.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147100K .......... .......... .......... .......... .......... 87% 13.2M 1s 2024-09-17T23:48:14.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147150K .......... .......... .......... .......... .......... 87% 47.2M 1s 2024-09-17T23:48:14.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147200K .......... .......... .......... .......... .......... 87% 40.9M 1s 2024-09-17T23:48:14.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147250K .......... .......... .......... .......... .......... 87% 19.7M 1s 2024-09-17T23:48:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147300K .......... .......... .......... .......... .......... 87% 25.9M 1s 2024-09-17T23:48:14.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147350K .......... .......... .......... .......... .......... 87% 21.1M 1s 2024-09-17T23:48:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147400K .......... .......... .......... .......... .......... 87% 40.9M 1s 2024-09-17T23:48:14.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147450K .......... .......... .......... .......... .......... 87% 20.7M 1s 2024-09-17T23:48:14.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147500K .......... .......... .......... .......... .......... 87% 27.8M 1s 2024-09-17T23:48:14.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147550K .......... .......... .......... .......... .......... 87% 16.9M 1s 2024-09-17T23:48:14.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147600K .......... .......... .......... .......... .......... 87% 21.9M 1s 2024-09-17T23:48:14.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147650K .......... .......... .......... .......... .......... 87% 19.2M 1s 2024-09-17T23:48:14.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147700K .......... .......... .......... .......... .......... 87% 53.0M 1s 2024-09-17T23:48:14.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147750K .......... .......... .......... .......... .......... 87% 21.9M 1s 2024-09-17T23:48:14.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147800K .......... .......... .......... .......... .......... 87% 58.5M 1s 2024-09-17T23:48:14.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147850K .......... .......... .......... .......... .......... 87% 21.5M 1s 2024-09-17T23:48:14.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147900K .......... .......... .......... .......... .......... 87% 42.2M 1s 2024-09-17T23:48:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:147950K .......... .......... .......... .......... .......... 87% 24.7M 1s 2024-09-17T23:48:14.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148000K .......... .......... .......... .......... .......... 87% 25.6M 1s 2024-09-17T23:48:14.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148050K .......... .......... .......... .......... .......... 87% 33.8M 1s 2024-09-17T23:48:14.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148100K .......... .......... .......... .......... .......... 87% 27.5M 1s 2024-09-17T23:48:14.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148150K .......... .......... .......... .......... .......... 87% 36.3M 1s 2024-09-17T23:48:14.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148200K .......... .......... .......... .......... .......... 87% 26.0M 1s 2024-09-17T23:48:14.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148250K .......... .......... .......... .......... .......... 87% 23.1M 1s 2024-09-17T23:48:14.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148300K .......... .......... .......... .......... .......... 88% 41.0M 1s 2024-09-17T23:48:14.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148350K .......... .......... .......... .......... .......... 88% 22.9M 1s 2024-09-17T23:48:14.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148400K .......... .......... .......... .......... .......... 88% 26.1M 1s 2024-09-17T23:48:14.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148450K .......... .......... .......... .......... .......... 88% 39.8M 1s 2024-09-17T23:48:14.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148500K .......... .......... .......... .......... .......... 88% 26.3M 1s 2024-09-17T23:48:14.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148550K .......... .......... .......... .......... .......... 88% 35.5M 1s 2024-09-17T23:48:14.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148600K .......... .......... .......... .......... .......... 88% 29.1M 1s 2024-09-17T23:48:14.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148650K .......... .......... .......... .......... .......... 88% 38.5M 1s 2024-09-17T23:48:14.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148700K .......... .......... .......... .......... .......... 88% 23.3M 1s 2024-09-17T23:48:14.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148750K .......... .......... .......... .......... .......... 88% 23.0M 1s 2024-09-17T23:48:14.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148800K .......... .......... .......... .......... .......... 88% 23.3M 1s 2024-09-17T23:48:14.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148850K .......... .......... .......... .......... .......... 88% 41.2M 1s 2024-09-17T23:48:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148900K .......... .......... .......... .......... .......... 88% 36.4M 1s 2024-09-17T23:48:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:148950K .......... .......... .......... .......... .......... 88% 28.3M 1s 2024-09-17T23:48:14.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149000K .......... .......... .......... .......... .......... 88% 25.1M 1s 2024-09-17T23:48:14.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149050K .......... .......... .......... .......... .......... 88% 21.2M 1s 2024-09-17T23:48:14.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149100K .......... .......... .......... .......... .......... 88% 27.4M 1s 2024-09-17T23:48:14.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149150K .......... .......... .......... .......... .......... 88% 23.2M 1s 2024-09-17T23:48:14.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149200K .......... .......... .......... .......... .......... 88% 35.4M 1s 2024-09-17T23:48:14.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149250K .......... .......... .......... .......... .......... 88% 20.9M 1s 2024-09-17T23:48:14.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149300K .......... .......... .......... .......... .......... 88% 32.9M 1s 2024-09-17T23:48:14.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149350K .......... .......... .......... .......... .......... 88% 26.7M 1s 2024-09-17T23:48:14.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149400K .......... .......... .......... .......... .......... 88% 24.2M 1s 2024-09-17T23:48:14.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149450K .......... .......... .......... .......... .......... 88% 44.4M 1s 2024-09-17T23:48:14.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149500K .......... .......... .......... .......... .......... 88% 28.4M 1s 2024-09-17T23:48:14.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149550K .......... .......... .......... .......... .......... 88% 31.4M 1s 2024-09-17T23:48:14.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149600K .......... .......... .......... .......... .......... 88% 25.9M 1s 2024-09-17T23:48:14.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149650K .......... .......... .......... .......... .......... 88% 25.5M 1s 2024-09-17T23:48:14.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149700K .......... .......... .......... .......... .......... 88% 39.7M 1s 2024-09-17T23:48:14.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149750K .......... .......... .......... .......... .......... 88% 20.5M 1s 2024-09-17T23:48:14.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149800K .......... .......... .......... .......... .......... 88% 43.8M 1s 2024-09-17T23:48:14.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149850K .......... .......... .......... .......... .......... 88% 22.6M 1s 2024-09-17T23:48:14.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149900K .......... .......... .......... .......... .......... 88% 30.8M 1s 2024-09-17T23:48:14.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:149950K .......... .......... .......... .......... .......... 88% 16.7M 1s 2024-09-17T23:48:14.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150000K .......... .......... .......... .......... .......... 89% 38.7M 1s 2024-09-17T23:48:14.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150050K .......... .......... .......... .......... .......... 89% 25.9M 1s 2024-09-17T23:48:14.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150100K .......... .......... .......... .......... .......... 89% 24.5M 1s 2024-09-17T23:48:14.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150150K .......... .......... .......... .......... .......... 89% 28.5M 1s 2024-09-17T23:48:14.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150200K .......... .......... .......... .......... .......... 89% 46.8M 1s 2024-09-17T23:48:14.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150250K .......... .......... .......... .......... .......... 89% 21.9M 1s 2024-09-17T23:48:14.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150300K .......... .......... .......... .......... .......... 89% 26.8M 1s 2024-09-17T23:48:14.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150350K .......... .......... .......... .......... .......... 89% 16.8M 1s 2024-09-17T23:48:14.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150400K .......... .......... .......... .......... .......... 89% 30.9M 1s 2024-09-17T23:48:14.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150450K .......... .......... .......... .......... .......... 89% 23.4M 1s 2024-09-17T23:48:14.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150500K .......... .......... .......... .......... .......... 89% 42.3M 1s 2024-09-17T23:48:14.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150550K .......... .......... .......... .......... .......... 89% 24.0M 1s 2024-09-17T23:48:14.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150600K .......... .......... .......... .......... .......... 89% 40.8M 1s 2024-09-17T23:48:14.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150650K .......... .......... .......... .......... .......... 89% 23.9M 1s 2024-09-17T23:48:14.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150700K .......... .......... .......... .......... .......... 89% 35.6M 1s 2024-09-17T23:48:14.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150750K .......... .......... .......... .......... .......... 89% 19.2M 1s 2024-09-17T23:48:14.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150800K .......... .......... .......... .......... .......... 89% 25.8M 1s 2024-09-17T23:48:14.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150850K .......... .......... .......... .......... .......... 89% 32.0M 1s 2024-09-17T23:48:14.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150900K .......... .......... .......... .......... .......... 89% 19.6M 1s 2024-09-17T23:48:14.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:150950K .......... .......... .......... .......... .......... 89% 39.6M 1s 2024-09-17T23:48:14.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151000K .......... .......... .......... .......... .......... 89% 17.1M 1s 2024-09-17T23:48:14.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151050K .......... .......... .......... .......... .......... 89% 25.7M 1s 2024-09-17T23:48:14.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151100K .......... .......... .......... .......... .......... 89% 22.5M 1s 2024-09-17T23:48:14.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151150K .......... .......... .......... .......... .......... 89% 22.7M 1s 2024-09-17T23:48:14.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151200K .......... .......... .......... .......... .......... 89% 23.7M 1s 2024-09-17T23:48:14.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151250K .......... .......... .......... .......... .......... 89% 25.8M 1s 2024-09-17T23:48:14.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151300K .......... .......... .......... .......... .......... 89% 23.4M 1s 2024-09-17T23:48:14.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151350K .......... .......... .......... .......... .......... 89% 22.2M 1s 2024-09-17T23:48:14.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151400K .......... .......... .......... .......... .......... 89% 28.1M 1s 2024-09-17T23:48:14.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151450K .......... .......... .......... .......... .......... 89% 35.2M 1s 2024-09-17T23:48:14.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151500K .......... .......... .......... .......... .......... 89% 23.1M 1s 2024-09-17T23:48:14.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151550K .......... .......... .......... .......... .......... 89% 23.9M 1s 2024-09-17T23:48:14.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151600K .......... .......... .......... .......... .......... 89% 33.0M 1s 2024-09-17T23:48:14.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151650K .......... .......... .......... .......... .......... 89% 25.4M 1s 2024-09-17T23:48:14.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151700K .......... .......... .......... .......... .......... 90% 25.1M 1s 2024-09-17T23:48:14.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151750K .......... .......... .......... .......... .......... 90% 42.0M 1s 2024-09-17T23:48:14.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151800K .......... .......... .......... .......... .......... 90% 24.9M 1s 2024-09-17T23:48:14.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151850K .......... .......... .......... .......... .......... 90% 38.0M 1s 2024-09-17T23:48:14.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151900K .......... .......... .......... .......... .......... 90% 25.6M 1s 2024-09-17T23:48:14.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:151950K .......... .......... .......... .......... .......... 90% 27.6M 1s 2024-09-17T23:48:14.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152000K .......... .......... .......... .......... .......... 90% 36.1M 1s 2024-09-17T23:48:14.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152050K .......... .......... .......... .......... .......... 90% 24.9M 1s 2024-09-17T23:48:14.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152100K .......... .......... .......... .......... .......... 90% 23.7M 1s 2024-09-17T23:48:14.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152150K .......... .......... .......... .......... .......... 90% 32.6M 1s 2024-09-17T23:48:14.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152200K .......... .......... .......... .......... .......... 90% 31.1M 1s 2024-09-17T23:48:14.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152250K .......... .......... .......... .......... .......... 90% 35.5M 1s 2024-09-17T23:48:14.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152300K .......... .......... .......... .......... .......... 90% 23.8M 1s 2024-09-17T23:48:14.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152350K .......... .......... .......... .......... .......... 90% 23.6M 1s 2024-09-17T23:48:14.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152400K .......... .......... .......... .......... .......... 90% 39.7M 1s 2024-09-17T23:48:14.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152450K .......... .......... .......... .......... .......... 90% 24.3M 1s 2024-09-17T23:48:14.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152500K .......... .......... .......... .......... .......... 90% 41.9M 1s 2024-09-17T23:48:14.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152550K .......... .......... .......... .......... .......... 90% 23.3M 1s 2024-09-17T23:48:14.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152600K .......... .......... .......... .......... .......... 90% 39.8M 1s 2024-09-17T23:48:14.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152650K .......... .......... .......... .......... .......... 90% 24.2M 1s 2024-09-17T23:48:14.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152700K .......... .......... .......... .......... .......... 90% 20.1M 1s 2024-09-17T23:48:14.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152750K .......... .......... .......... .......... .......... 90% 23.9M 1s 2024-09-17T23:48:14.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152800K .......... .......... .......... .......... .......... 90% 34.8M 1s 2024-09-17T23:48:14.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152850K .......... .......... .......... .......... .......... 90% 22.0M 1s 2024-09-17T23:48:14.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152900K .......... .......... .......... .......... .......... 90% 30.0M 1s 2024-09-17T23:48:14.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:152950K .......... .......... .......... .......... .......... 90% 24.1M 1s 2024-09-17T23:48:14.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153000K .......... .......... .......... .......... .......... 90% 42.9M 1s 2024-09-17T23:48:14.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153050K .......... .......... .......... .......... .......... 90% 24.1M 1s 2024-09-17T23:48:14.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153100K .......... .......... .......... .......... .......... 90% 43.4M 1s 2024-09-17T23:48:14.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153150K .......... .......... .......... .......... .......... 90% 24.6M 1s 2024-09-17T23:48:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153200K .......... .......... .......... .......... .......... 90% 23.8M 1s 2024-09-17T23:48:14.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153250K .......... .......... .......... .......... .......... 90% 38.1M 1s 2024-09-17T23:48:14.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153300K .......... .......... .......... .......... .......... 90% 23.3M 1s 2024-09-17T23:48:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153350K .......... .......... .......... .......... .......... 91% 21.3M 1s 2024-09-17T23:48:14.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153400K .......... .......... .......... .......... .......... 91% 41.3M 1s 2024-09-17T23:48:14.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153450K .......... .......... .......... .......... .......... 91% 23.2M 1s 2024-09-17T23:48:14.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153500K .......... .......... .......... .......... .......... 91% 40.6M 1s 2024-09-17T23:48:14.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153550K .......... .......... .......... .......... .......... 91% 20.4M 1s 2024-09-17T23:48:14.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153600K .......... .......... .......... .......... .......... 91% 24.8M 1s 2024-09-17T23:48:14.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153650K .......... .......... .......... .......... .......... 91% 45.0M 1s 2024-09-17T23:48:14.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153700K .......... .......... .......... .......... .......... 91% 22.9M 1s 2024-09-17T23:48:14.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153750K .......... .......... .......... .......... .......... 91% 39.6M 1s 2024-09-17T23:48:14.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153800K .......... .......... .......... .......... .......... 91% 21.0M 1s 2024-09-17T23:48:14.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153850K .......... .......... .......... .......... .......... 91% 26.0M 1s 2024-09-17T23:48:14.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153900K .......... .......... .......... .......... .......... 91% 27.2M 1s 2024-09-17T23:48:14.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:153950K .......... .......... .......... .......... .......... 91% 22.3M 1s 2024-09-17T23:48:14.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154000K .......... .......... .......... .......... .......... 91% 18.5M 1s 2024-09-17T23:48:14.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154050K .......... .......... .......... .......... .......... 91% 38.7M 1s 2024-09-17T23:48:14.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154100K .......... .......... .......... .......... .......... 91% 23.4M 1s 2024-09-17T23:48:14.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154150K .......... .......... .......... .......... .......... 91% 42.4M 1s 2024-09-17T23:48:14.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154200K .......... .......... .......... .......... .......... 91% 26.3M 1s 2024-09-17T23:48:14.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154250K .......... .......... .......... .......... .......... 91% 38.8M 1s 2024-09-17T23:48:14.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154300K .......... .......... .......... .......... .......... 91% 23.7M 1s 2024-09-17T23:48:14.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154350K .......... .......... .......... .......... .......... 91% 20.9M 0s 2024-09-17T23:48:14.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154400K .......... .......... .......... .......... .......... 91% 29.8M 0s 2024-09-17T23:48:14.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154450K .......... .......... .......... .......... .......... 91% 21.5M 0s 2024-09-17T23:48:14.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154500K .......... .......... .......... .......... .......... 91% 21.3M 0s 2024-09-17T23:48:14.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154550K .......... .......... .......... .......... .......... 91% 38.8M 0s 2024-09-17T23:48:14.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154600K .......... .......... .......... .......... .......... 91% 20.5M 0s 2024-09-17T23:48:14.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154650K .......... .......... .......... .......... .......... 91% 39.0M 0s 2024-09-17T23:48:14.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154700K .......... .......... .......... .......... .......... 91% 16.0M 0s 2024-09-17T23:48:14.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154750K .......... .......... .......... .......... .......... 91% 9.55M 0s 2024-09-17T23:48:14.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154800K .......... .......... .......... .......... .......... 91% 78.5M 0s 2024-09-17T23:48:14.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154850K .......... .......... .......... .......... .......... 91% 85.7M 0s 2024-09-17T23:48:14.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154900K .......... .......... .......... .......... .......... 91% 73.4M 0s 2024-09-17T23:48:14.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:154950K .......... .......... .......... .......... .......... 91% 21.7M 0s 2024-09-17T23:48:14.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155000K .......... .......... .......... .......... .......... 91% 19.1M 0s 2024-09-17T23:48:14.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155050K .......... .......... .......... .......... .......... 92% 35.1M 0s 2024-09-17T23:48:14.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155100K .......... .......... .......... .......... .......... 92% 18.0M 0s 2024-09-17T23:48:14.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155150K .......... .......... .......... .......... .......... 92% 22.1M 0s 2024-09-17T23:48:14.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155200K .......... .......... .......... .......... .......... 92% 29.9M 0s 2024-09-17T23:48:14.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155250K .......... .......... .......... .......... .......... 92% 22.7M 0s 2024-09-17T23:48:14.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155300K .......... .......... .......... .......... .......... 92% 48.0M 0s 2024-09-17T23:48:14.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155350K .......... .......... .......... .......... .......... 92% 23.6M 0s 2024-09-17T23:48:14.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155400K .......... .......... .......... .......... .......... 92% 39.7M 0s 2024-09-17T23:48:14.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155450K .......... .......... .......... .......... .......... 92% 24.7M 0s 2024-09-17T23:48:14.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155500K .......... .......... .......... .......... .......... 92% 44.0M 0s 2024-09-17T23:48:14.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155550K .......... .......... .......... .......... .......... 92% 16.3M 0s 2024-09-17T23:48:14.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155600K .......... .......... .......... .......... .......... 92% 43.8M 0s 2024-09-17T23:48:14.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155650K .......... .......... .......... .......... .......... 92% 27.0M 0s 2024-09-17T23:48:14.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155700K .......... .......... .......... .......... .......... 92% 35.9M 0s 2024-09-17T23:48:14.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155750K .......... .......... .......... .......... .......... 92% 23.9M 0s 2024-09-17T23:48:14.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155800K .......... .......... .......... .......... .......... 92% 44.4M 0s 2024-09-17T23:48:14.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155850K .......... .......... .......... .......... .......... 92% 25.5M 0s 2024-09-17T23:48:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155900K .......... .......... .......... .......... .......... 92% 35.5M 0s 2024-09-17T23:48:14.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:155950K .......... .......... .......... .......... .......... 92% 23.5M 0s 2024-09-17T23:48:14.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156000K .......... .......... .......... .......... .......... 92% 25.5M 0s 2024-09-17T23:48:14.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156050K .......... .......... .......... .......... .......... 92% 41.8M 0s 2024-09-17T23:48:14.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156100K .......... .......... .......... .......... .......... 92% 25.7M 0s 2024-09-17T23:48:14.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156150K .......... .......... .......... .......... .......... 92% 20.3M 0s 2024-09-17T23:48:14.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156200K .......... .......... .......... .......... .......... 92% 40.3M 0s 2024-09-17T23:48:14.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156250K .......... .......... .......... .......... .......... 92% 24.6M 0s 2024-09-17T23:48:14.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156300K .......... .......... .......... .......... .......... 92% 42.4M 0s 2024-09-17T23:48:14.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156350K .......... .......... .......... .......... .......... 92% 22.1M 0s 2024-09-17T23:48:14.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156400K .......... .......... .......... .......... .......... 92% 17.0M 0s 2024-09-17T23:48:14.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156450K .......... .......... .......... .......... .......... 92% 39.6M 0s 2024-09-17T23:48:14.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156500K .......... .......... .......... .......... .......... 92% 24.5M 0s 2024-09-17T23:48:14.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156550K .......... .......... .......... .......... .......... 92% 37.4M 0s 2024-09-17T23:48:14.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156600K .......... .......... .......... .......... .......... 92% 22.7M 0s 2024-09-17T23:48:14.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156650K .......... .......... .......... .......... .......... 92% 35.5M 0s 2024-09-17T23:48:14.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156700K .......... .......... .......... .......... .......... 92% 24.8M 0s 2024-09-17T23:48:14.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156750K .......... .......... .......... .......... .......... 93% 25.6M 0s 2024-09-17T23:48:14.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156800K .......... .......... .......... .......... .......... 93% 27.6M 0s 2024-09-17T23:48:14.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156850K .......... .......... .......... .......... .......... 93% 39.1M 0s 2024-09-17T23:48:14.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156900K .......... .......... .......... .......... .......... 93% 26.0M 0s 2024-09-17T23:48:14.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:156950K .......... .......... .......... .......... .......... 93% 27.7M 0s 2024-09-17T23:48:14.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157000K .......... .......... .......... .......... .......... 93% 21.9M 0s 2024-09-17T23:48:14.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157050K .......... .......... .......... .......... .......... 93% 53.4M 0s 2024-09-17T23:48:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157100K .......... .......... .......... .......... .......... 93% 21.5M 0s 2024-09-17T23:48:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157150K .......... .......... .......... .......... .......... 93% 26.8M 0s 2024-09-17T23:48:14.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157200K .......... .......... .......... .......... .......... 93% 25.5M 0s 2024-09-17T23:48:14.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157250K .......... .......... .......... .......... .......... 93% 21.9M 0s 2024-09-17T23:48:14.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157300K .......... .......... .......... .......... .......... 93% 24.8M 0s 2024-09-17T23:48:14.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157350K .......... .......... .......... .......... .......... 93% 40.1M 0s 2024-09-17T23:48:14.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157400K .......... .......... .......... .......... .......... 93% 25.0M 0s 2024-09-17T23:48:14.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157450K .......... .......... .......... .......... .......... 93% 31.3M 0s 2024-09-17T23:48:14.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157500K .......... .......... .......... .......... .......... 93% 23.3M 0s 2024-09-17T23:48:14.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157550K .......... .......... .......... .......... .......... 93% 21.9M 0s 2024-09-17T23:48:14.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157600K .......... .......... .......... .......... .......... 93% 38.9M 0s 2024-09-17T23:48:14.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157650K .......... .......... .......... .......... .......... 93% 23.7M 0s 2024-09-17T23:48:14.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157700K .......... .......... .......... .......... .......... 93% 39.8M 0s 2024-09-17T23:48:14.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157750K .......... .......... .......... .......... .......... 93% 14.9M 0s 2024-09-17T23:48:14.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157800K .......... .......... .......... .......... .......... 93% 29.2M 0s 2024-09-17T23:48:14.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157850K .......... .......... .......... .......... .......... 93% 40.8M 0s 2024-09-17T23:48:14.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157900K .......... .......... .......... .......... .......... 93% 28.0M 0s 2024-09-17T23:48:14.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:157950K .......... .......... .......... .......... .......... 93% 20.5M 0s 2024-09-17T23:48:14.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158000K .......... .......... .......... .......... .......... 93% 27.5M 0s 2024-09-17T23:48:14.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158050K .......... .......... .......... .......... .......... 93% 26.1M 0s 2024-09-17T23:48:14.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158100K .......... .......... .......... .......... .......... 93% 35.0M 0s 2024-09-17T23:48:14.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158150K .......... .......... .......... .......... .......... 93% 23.0M 0s 2024-09-17T23:48:14.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158200K .......... .......... .......... .......... .......... 93% 36.8M 0s 2024-09-17T23:48:14.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158250K .......... .......... .......... .......... .......... 93% 23.8M 0s 2024-09-17T23:48:14.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158300K .......... .......... .......... .......... .......... 93% 29.0M 0s 2024-09-17T23:48:14.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158350K .......... .......... .......... .......... .......... 93% 14.5M 0s 2024-09-17T23:48:14.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158400K .......... .......... .......... .......... .......... 93% 56.3M 0s 2024-09-17T23:48:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158450K .......... .......... .......... .......... .......... 94% 16.2M 0s 2024-09-17T23:48:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158500K .......... .......... .......... .......... .......... 94% 37.4M 0s 2024-09-17T23:48:14.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158550K .......... .......... .......... .......... .......... 94% 21.3M 0s 2024-09-17T23:48:14.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158600K .......... .......... .......... .......... .......... 94% 34.7M 0s 2024-09-17T23:48:14.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158650K .......... .......... .......... .......... .......... 94% 23.0M 0s 2024-09-17T23:48:14.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158700K .......... .......... .......... .......... .......... 94% 34.9M 0s 2024-09-17T23:48:14.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158750K .......... .......... .......... .......... .......... 94% 19.4M 0s 2024-09-17T23:48:14.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158800K .......... .......... .......... .......... .......... 94% 20.9M 0s 2024-09-17T23:48:14.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158850K .......... .......... .......... .......... .......... 94% 33.1M 0s 2024-09-17T23:48:14.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158900K .......... .......... .......... .......... .......... 94% 20.0M 0s 2024-09-17T23:48:14.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:158950K .......... .......... .......... .......... .......... 94% 38.7M 0s 2024-09-17T23:48:14.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159000K .......... .......... .......... .......... .......... 94% 18.3M 0s 2024-09-17T23:48:14.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159050K .......... .......... .......... .......... .......... 94% 24.1M 0s 2024-09-17T23:48:14.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159100K .......... .......... .......... .......... .......... 94% 40.8M 0s 2024-09-17T23:48:14.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159150K .......... .......... .......... .......... .......... 94% 24.2M 0s 2024-09-17T23:48:14.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159200K .......... .......... .......... .......... .......... 94% 25.6M 0s 2024-09-17T23:48:14.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159250K .......... .......... .......... .......... .......... 94% 36.4M 0s 2024-09-17T23:48:14.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159300K .......... .......... .......... .......... .......... 94% 25.8M 0s 2024-09-17T23:48:14.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159350K .......... .......... .......... .......... .......... 94% 38.8M 0s 2024-09-17T23:48:14.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159400K .......... .......... .......... .......... .......... 94% 24.3M 0s 2024-09-17T23:48:14.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159450K .......... .......... .......... .......... .......... 94% 42.5M 0s 2024-09-17T23:48:14.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159500K .......... .......... .......... .......... .......... 94% 25.1M 0s 2024-09-17T23:48:14.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159550K .......... .......... .......... .......... .......... 94% 22.4M 0s 2024-09-17T23:48:14.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159600K .......... .......... .......... .......... .......... 94% 24.7M 0s 2024-09-17T23:48:14.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159650K .......... .......... .......... .......... .......... 94% 40.1M 0s 2024-09-17T23:48:14.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159700K .......... .......... .......... .......... .......... 94% 25.1M 0s 2024-09-17T23:48:14.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159750K .......... .......... .......... .......... .......... 94% 37.5M 0s 2024-09-17T23:48:14.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159800K .......... .......... .......... .......... .......... 94% 25.8M 0s 2024-09-17T23:48:14.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159850K .......... .......... .......... .......... .......... 94% 38.9M 0s 2024-09-17T23:48:14.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159900K .......... .......... .......... .......... .......... 94% 23.7M 0s 2024-09-17T23:48:14.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:159950K .......... .......... .......... .......... .......... 94% 24.6M 0s 2024-09-17T23:48:14.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160000K .......... .......... .......... .......... .......... 94% 39.1M 0s 2024-09-17T23:48:14.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160050K .......... .......... .......... .......... .......... 94% 23.2M 0s 2024-09-17T23:48:14.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160100K .......... .......... .......... .......... .......... 95% 15.9M 0s 2024-09-17T23:48:14.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160150K .......... .......... .......... .......... .......... 95% 39.1M 0s 2024-09-17T23:48:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160200K .......... .......... .......... .......... .......... 95% 22.6M 0s 2024-09-17T23:48:14.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160250K .......... .......... .......... .......... .......... 95% 41.9M 0s 2024-09-17T23:48:14.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160300K .......... .......... .......... .......... .......... 95% 21.9M 0s 2024-09-17T23:48:14.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160350K .......... .......... .......... .......... .......... 95% 25.1M 0s 2024-09-17T23:48:14.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160400K .......... .......... .......... .......... .......... 95% 38.3M 0s 2024-09-17T23:48:14.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160450K .......... .......... .......... .......... .......... 95% 28.0M 0s 2024-09-17T23:48:14.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160500K .......... .......... .......... .......... .......... 95% 24.5M 0s 2024-09-17T23:48:14.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160550K .......... .......... .......... .......... .......... 95% 32.2M 0s 2024-09-17T23:48:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160600K .......... .......... .......... .......... .......... 95% 48.3M 0s 2024-09-17T23:48:14.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160650K .......... .......... .......... .......... .......... 95% 23.4M 0s 2024-09-17T23:48:14.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160700K .......... .......... .......... .......... .......... 95% 20.8M 0s 2024-09-17T23:48:14.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160750K .......... .......... .......... .......... .......... 95% 24.0M 0s 2024-09-17T23:48:14.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160800K .......... .......... .......... .......... .......... 95% 40.1M 0s 2024-09-17T23:48:14.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160850K .......... .......... .......... .......... .......... 95% 19.4M 0s 2024-09-17T23:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160900K .......... .......... .......... .......... .......... 95% 41.9M 0s 2024-09-17T23:48:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:160950K .......... .......... .......... .......... .......... 95% 23.0M 0s 2024-09-17T23:48:14.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161000K .......... .......... .......... .......... .......... 95% 43.0M 0s 2024-09-17T23:48:14.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161050K .......... .......... .......... .......... .......... 95% 23.8M 0s 2024-09-17T23:48:14.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161100K .......... .......... .......... .......... .......... 95% 36.0M 0s 2024-09-17T23:48:14.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161150K .......... .......... .......... .......... .......... 95% 20.7M 0s 2024-09-17T23:48:14.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161200K .......... .......... .......... .......... .......... 95% 20.8M 0s 2024-09-17T23:48:14.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161250K .......... .......... .......... .......... .......... 95% 22.0M 0s 2024-09-17T23:48:14.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161300K .......... .......... .......... .......... .......... 95% 36.7M 0s 2024-09-17T23:48:14.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161350K .......... .......... .......... .......... .......... 95% 24.7M 0s 2024-09-17T23:48:14.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161400K .......... .......... .......... .......... .......... 95% 34.1M 0s 2024-09-17T23:48:14.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161450K .......... .......... .......... .......... .......... 95% 20.9M 0s 2024-09-17T23:48:14.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161500K .......... .......... .......... .......... .......... 95% 31.2M 0s 2024-09-17T23:48:14.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161550K .......... .......... .......... .......... .......... 95% 23.8M 0s 2024-09-17T23:48:14.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161600K .......... .......... .......... .......... .......... 95% 25.3M 0s 2024-09-17T23:48:14.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161650K .......... .......... .......... .......... .......... 95% 40.4M 0s 2024-09-17T23:48:14.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161700K .......... .......... .......... .......... .......... 95% 21.9M 0s 2024-09-17T23:48:14.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161750K .......... .......... .......... .......... .......... 95% 32.7M 0s 2024-09-17T23:48:14.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161800K .......... .......... .......... .......... .......... 96% 20.0M 0s 2024-09-17T23:48:14.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161850K .......... .......... .......... .......... .......... 96% 24.5M 0s 2024-09-17T23:48:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161900K .......... .......... .......... .......... .......... 96% 45.7M 0s 2024-09-17T23:48:14.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:161950K .......... .......... .......... .......... .......... 96% 18.9M 0s 2024-09-17T23:48:14.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162000K .......... .......... .......... .......... .......... 96% 21.4M 0s 2024-09-17T23:48:14.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162050K .......... .......... .......... .......... .......... 96% 36.0M 0s 2024-09-17T23:48:14.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162100K .......... .......... .......... .......... .......... 96% 22.5M 0s 2024-09-17T23:48:14.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162150K .......... .......... .......... .......... .......... 96% 36.3M 0s 2024-09-17T23:48:14.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162200K .......... .......... .......... .......... .......... 96% 18.0M 0s 2024-09-17T23:48:14.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162250K .......... .......... .......... .......... .......... 96% 39.8M 0s 2024-09-17T23:48:14.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162300K .......... .......... .......... .......... .......... 96% 21.6M 0s 2024-09-17T23:48:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162350K .......... .......... .......... .......... .......... 96% 18.8M 0s 2024-09-17T23:48:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162400K .......... .......... .......... .......... .......... 96% 22.8M 0s 2024-09-17T23:48:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162450K .......... .......... .......... .......... .......... 96% 44.4M 0s 2024-09-17T23:48:14.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162500K .......... .......... .......... .......... .......... 96% 18.4M 0s 2024-09-17T23:48:14.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162550K .......... .......... .......... .......... .......... 96% 33.9M 0s 2024-09-17T23:48:14.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162600K .......... .......... .......... .......... .......... 96% 20.3M 0s 2024-09-17T23:48:14.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162650K .......... .......... .......... .......... .......... 96% 37.2M 0s 2024-09-17T23:48:14.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162700K .......... .......... .......... .......... .......... 96% 21.2M 0s 2024-09-17T23:48:14.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162750K .......... .......... .......... .......... .......... 96% 20.3M 0s 2024-09-17T23:48:14.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162800K .......... .......... .......... .......... .......... 96% 31.0M 0s 2024-09-17T23:48:14.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162850K .......... .......... .......... .......... .......... 96% 33.0M 0s 2024-09-17T23:48:14.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162900K .......... .......... .......... .......... .......... 96% 19.6M 0s 2024-09-17T23:48:14.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:162950K .......... .......... .......... .......... .......... 96% 44.9M 0s 2024-09-17T23:48:14.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163000K .......... .......... .......... .......... .......... 96% 23.2M 0s 2024-09-17T23:48:14.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163050K .......... .......... .......... .......... .......... 96% 41.1M 0s 2024-09-17T23:48:14.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163100K .......... .......... .......... .......... .......... 96% 25.8M 0s 2024-09-17T23:48:14.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163150K .......... .......... .......... .......... .......... 96% 23.3M 0s 2024-09-17T23:48:14.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163200K .......... .......... .......... .......... .......... 96% 42.9M 0s 2024-09-17T23:48:14.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163250K .......... .......... .......... .......... .......... 96% 24.8M 0s 2024-09-17T23:48:14.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163300K .......... .......... .......... .......... .......... 96% 39.7M 0s 2024-09-17T23:48:14.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163350K .......... .......... .......... .......... .......... 96% 22.9M 0s 2024-09-17T23:48:14.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163400K .......... .......... .......... .......... .......... 96% 42.0M 0s 2024-09-17T23:48:14.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163450K .......... .......... .......... .......... .......... 96% 24.7M 0s 2024-09-17T23:48:14.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163500K .......... .......... .......... .......... .......... 97% 36.6M 0s 2024-09-17T23:48:14.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163550K .......... .......... .......... .......... .......... 97% 16.3M 0s 2024-09-17T23:48:14.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163600K .......... .......... .......... .......... .......... 97% 40.4M 0s 2024-09-17T23:48:14.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163650K .......... .......... .......... .......... .......... 97% 24.7M 0s 2024-09-17T23:48:14.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163700K .......... .......... .......... .......... .......... 97% 44.0M 0s 2024-09-17T23:48:14.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163750K .......... .......... .......... .......... .......... 97% 20.7M 0s 2024-09-17T23:48:14.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163800K .......... .......... .......... .......... .......... 97% 23.4M 0s 2024-09-17T23:48:14.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163850K .......... .......... .......... .......... .......... 97% 24.4M 0s 2024-09-17T23:48:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163900K .......... .......... .......... .......... .......... 97% 41.5M 0s 2024-09-17T23:48:14.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:163950K .......... .......... .......... .......... .......... 97% 20.1M 0s 2024-09-17T23:48:14.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164000K .......... .......... .......... .......... .......... 97% 23.6M 0s 2024-09-17T23:48:14.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164050K .......... .......... .......... .......... .......... 97% 23.4M 0s 2024-09-17T23:48:14.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164100K .......... .......... .......... .......... .......... 97% 42.1M 0s 2024-09-17T23:48:14.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164150K .......... .......... .......... .......... .......... 97% 24.3M 0s 2024-09-17T23:48:14.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164200K .......... .......... .......... .......... .......... 97% 39.3M 0s 2024-09-17T23:48:14.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164250K .......... .......... .......... .......... .......... 97% 24.8M 0s 2024-09-17T23:48:14.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164300K .......... .......... .......... .......... .......... 97% 32.1M 0s 2024-09-17T23:48:14.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164350K .......... .......... .......... .......... .......... 97% 23.0M 0s 2024-09-17T23:48:14.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164400K .......... .......... .......... .......... .......... 97% 24.9M 0s 2024-09-17T23:48:14.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164450K .......... .......... .......... .......... .......... 97% 31.4M 0s 2024-09-17T23:48:14.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164500K .......... .......... .......... .......... .......... 97% 21.7M 0s 2024-09-17T23:48:14.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164550K .......... .......... .......... .......... .......... 97% 38.9M 0s 2024-09-17T23:48:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164600K .......... .......... .......... .......... .......... 97% 25.1M 0s 2024-09-17T23:48:14.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164650K .......... .......... .......... .......... .......... 97% 23.9M 0s 2024-09-17T23:48:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164700K .......... .......... .......... .......... .......... 97% 42.5M 0s 2024-09-17T23:48:14.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164750K .......... .......... .......... .......... .......... 97% 23.3M 0s 2024-09-17T23:48:14.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164800K .......... .......... .......... .......... .......... 97% 24.8M 0s 2024-09-17T23:48:14.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164850K .......... .......... .......... .......... .......... 97% 41.7M 0s 2024-09-17T23:48:14.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164900K .......... .......... .......... .......... .......... 97% 24.7M 0s 2024-09-17T23:48:14.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:164950K .......... .......... .......... .......... .......... 97% 40.9M 0s 2024-09-17T23:48:14.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165000K .......... .......... .......... .......... .......... 97% 24.5M 0s 2024-09-17T23:48:14.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165050K .......... .......... .......... .......... .......... 97% 41.3M 0s 2024-09-17T23:48:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165100K .......... .......... .......... .......... .......... 97% 25.2M 0s 2024-09-17T23:48:14.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165150K .......... .......... .......... .......... .......... 98% 24.5M 0s 2024-09-17T23:48:14.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165200K .......... .......... .......... .......... .......... 98% 23.1M 0s 2024-09-17T23:48:14.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165250K .......... .......... .......... .......... .......... 98% 38.7M 0s 2024-09-17T23:48:14.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165300K .......... .......... .......... .......... .......... 98% 25.7M 0s 2024-09-17T23:48:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165350K .......... .......... .......... .......... .......... 98% 39.0M 0s 2024-09-17T23:48:14.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165400K .......... .......... .......... .......... .......... 98% 25.3M 0s 2024-09-17T23:48:14.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165450K .......... .......... .......... .......... .......... 98% 42.6M 0s 2024-09-17T23:48:14.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165500K .......... .......... .......... .......... .......... 98% 24.5M 0s 2024-09-17T23:48:14.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165550K .......... .......... .......... .......... .......... 98% 23.9M 0s 2024-09-17T23:48:14.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165600K .......... .......... .......... .......... .......... 98% 40.4M 0s 2024-09-17T23:48:14.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165650K .......... .......... .......... .......... .......... 98% 25.1M 0s 2024-09-17T23:48:14.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165700K .......... .......... .......... .......... .......... 98% 46.9M 0s 2024-09-17T23:48:14.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165750K .......... .......... .......... .......... .......... 98% 21.5M 0s 2024-09-17T23:48:14.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165800K .......... .......... .......... .......... .......... 98% 25.3M 0s 2024-09-17T23:48:14.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165850K .......... .......... .......... .......... .......... 98% 39.9M 0s 2024-09-17T23:48:14.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165900K .......... .......... .......... .......... .......... 98% 24.5M 0s 2024-09-17T23:48:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:165950K .......... .......... .......... .......... .......... 98% 24.5M 0s 2024-09-17T23:48:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166000K .......... .......... .......... .......... .......... 98% 38.9M 0s 2024-09-17T23:48:14.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166050K .......... .......... .......... .......... .......... 98% 19.8M 0s 2024-09-17T23:48:14.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166100K .......... .......... .......... .......... .......... 98% 76.1M 0s 2024-09-17T23:48:14.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166150K .......... .......... .......... .......... .......... 98% 23.0M 0s 2024-09-17T23:48:14.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166200K .......... .......... .......... .......... .......... 98% 32.8M 0s 2024-09-17T23:48:14.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166250K .......... .......... .......... .......... .......... 98% 29.7M 0s 2024-09-17T23:48:14.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166300K .......... .......... .......... .......... .......... 98% 46.3M 0s 2024-09-17T23:48:14.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166350K .......... .......... .......... .......... .......... 98% 7.12M 0s 2024-09-17T23:48:14.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166400K .......... .......... .......... .......... .......... 98% 83.7M 0s 2024-09-17T23:48:14.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166450K .......... .......... .......... .......... .......... 98% 88.4M 0s 2024-09-17T23:48:14.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166500K .......... .......... .......... .......... .......... 98% 78.9M 0s 2024-09-17T23:48:14.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166550K .......... .......... .......... .......... .......... 98% 79.6M 0s 2024-09-17T23:48:14.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166600K .......... .......... .......... .......... .......... 98% 37.6M 0s 2024-09-17T23:48:14.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166650K .......... .......... .......... .......... .......... 98% 27.0M 0s 2024-09-17T23:48:14.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166700K .......... .......... .......... .......... .......... 98% 44.3M 0s 2024-09-17T23:48:14.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166750K .......... .......... .......... .......... .......... 98% 23.5M 0s 2024-09-17T23:48:14.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166800K .......... .......... .......... .......... .......... 98% 24.7M 0s 2024-09-17T23:48:14.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166850K .......... .......... .......... .......... .......... 99% 38.1M 0s 2024-09-17T23:48:15.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166900K .......... .......... .......... .......... .......... 99% 24.2M 0s 2024-09-17T23:48:15.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:166950K .......... .......... .......... .......... .......... 99% 24.6M 0s 2024-09-17T23:48:15.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167000K .......... .......... .......... .......... .......... 99% 45.3M 0s 2024-09-17T23:48:15.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167050K .......... .......... .......... .......... .......... 99% 23.8M 0s 2024-09-17T23:48:15.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167100K .......... .......... .......... .......... .......... 99% 41.5M 0s 2024-09-17T23:48:15.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167150K .......... .......... .......... .......... .......... 99% 21.5M 0s 2024-09-17T23:48:15.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167200K .......... .......... .......... .......... .......... 99% 26.3M 0s 2024-09-17T23:48:15.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167250K .......... .......... .......... .......... .......... 99% 42.8M 0s 2024-09-17T23:48:15.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167300K .......... .......... .......... .......... .......... 99% 26.1M 0s 2024-09-17T23:48:15.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167350K .......... .......... .......... .......... .......... 99% 38.6M 0s 2024-09-17T23:48:15.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167400K .......... .......... .......... .......... .......... 99% 23.2M 0s 2024-09-17T23:48:15.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167450K .......... .......... .......... .......... .......... 99% 23.6M 0s 2024-09-17T23:48:15.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167500K .......... .......... .......... .......... .......... 99% 52.4M 0s 2024-09-17T23:48:15.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167550K .......... .......... .......... .......... .......... 99% 22.0M 0s 2024-09-17T23:48:15.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167600K .......... .......... .......... .......... .......... 99% 26.0M 0s 2024-09-17T23:48:15.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167650K .......... .......... .......... .......... .......... 99% 37.6M 0s 2024-09-17T23:48:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167700K .......... .......... .......... .......... .......... 99% 24.8M 0s 2024-09-17T23:48:15.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167750K .......... .......... .......... .......... .......... 99% 43.8M 0s 2024-09-17T23:48:15.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167800K .......... .......... .......... .......... .......... 99% 24.6M 0s 2024-09-17T23:48:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167850K .......... .......... .......... .......... .......... 99% 37.4M 0s 2024-09-17T23:48:15.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167900K .......... .......... .......... .......... .......... 99% 21.5M 0s 2024-09-17T23:48:15.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:167950K .......... .......... .......... .......... .......... 99% 29.6M 0s 2024-09-17T23:48:15.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:168000K .......... .......... .......... .......... .......... 99% 23.1M 0s 2024-09-17T23:48:15.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:168050K .......... .......... .......... .......... .......... 99% 49.2M 0s 2024-09-17T23:48:15.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:168100K .......... .......... .......... .......... .......... 99% 23.7M 0s 2024-09-17T23:48:15.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:168150K .......... .......... .......... .......... .......... 99% 38.6M 0s 2024-09-17T23:48:15.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:168200K .......... .......... .......... .......... .......... 99% 25.4M 0s 2024-09-17T23:48:15.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:168250K .......... .......... .......... .......... .......... 99% 43.2M 0s 2024-09-17T23:48:15.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:168300K .......... .......... .......... .......... .......... 99% 35.9M 0s 2024-09-17T23:48:15.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:168350K .......... .......... .......... .......... .......... 99% 23.8M 0s 2024-09-17T23:48:15.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:168400K .......... .......... .......... .......... .......... 99% 22.9M 0s 2024-09-17T23:48:15.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:168450K .......... .......... .......... .......... .......... 99% 29.0M 0s 2024-09-17T23:48:15.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:168500K .......... .......... .......... .......... .......... 99% 54.1M 0s 2024-09-17T23:48:15.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:168550K .......... .......... . 100% 59.2M=5.9s 2024-09-17T23:48:15.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2024-09-17T23:48:15.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:2024-09-17 23:48:15 (27.7 MB/s) - 'linux-5.4.tar.gz' saved [172616875/172616875] 2024-09-17T23:48:15.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: 2024-09-17T23:48:15.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..10} 2024-09-17T23:48:15.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS=() 2024-09-17T23:48:15.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS=() 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS=() 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS=() 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RET=0 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary1 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary1 snapshot 10G 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:48:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary1 10G 2024-09-17T23:48:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:48:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:48:15.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:48:15.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:15.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary1 2024-09-17T23:48:15.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:48:15.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:15.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:48:15.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:15.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary1 snapshot 2024-09-17T23:48:15.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:15.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:15.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:15.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:15.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary1 snapshot 2024-09-17T23:48:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:48:15.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary1 2024-09-17T23:48:15.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary1': 2024-09-17T23:48:15.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:48:15.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:48:15.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:48:15.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 10acdf255f6c 2024-09-17T23:48:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.10acdf255f6c 2024-09-17T23:48:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:48:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:48:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:48:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:48:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:48:15 2024 2024-09-17T23:48:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:48:15 2024 2024-09-17T23:48:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:48:15 2024 2024-09-17T23:48:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:48:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:48:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 52d5d797-77fc-4c42-8bb6-0d80760c93da 2024-09-17T23:48:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:48:15.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:48:15.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-17T23:48:16.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd0 2024-09-17T23:48:16.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:48:16.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd0 2024-09-17T23:48:16.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:48:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:48:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:48:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: a6a21d7c-1138-4791-9111-e44dbc9b7da7 2024-09-17T23:48:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:48:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:48:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:48:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:48:16.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:48:16.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:48:16.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:16.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary1 2024-09-17T23:48:16.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd0 test-primary1 2024-09-17T23:48:16.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:48:16.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary1 2024-09-17T23:48:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:48:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary1 2024-09-17T23:48:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:48:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:48:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:48:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:16.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary1 2024-09-17T23:48:16.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:16.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:16.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-17T23:48:16.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary2 2024-09-17T23:48:16.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:48:16.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary2 snapshot 10G 2024-09-17T23:48:16.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:16.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary1 2024-09-17T23:48:16.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:16.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:16.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:16.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary1 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary2 10G 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:48:16.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:48:16.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:48:16.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:16.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary2 2024-09-17T23:48:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:48:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:16.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:48:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary2 snapshot 2024-09-17T23:48:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:16.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary2 snapshot 2024-09-17T23:48:16.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:48:16.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary1/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary1' 2024-09-17T23:48:16.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:48:16.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 4 2024-09-17T23:48:16.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:16.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary2 2024-09-17T23:48:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary2': 2024-09-17T23:48:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:48:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:48:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:48:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 10bcd24618af 2024-09-17T23:48:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.10bcd24618af 2024-09-17T23:48:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:48:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:48:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:48:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:48:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:48:16 2024 2024-09-17T23:48:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:48:16 2024 2024-09-17T23:48:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:48:16 2024 2024-09-17T23:48:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:48:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:48:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 8f9eb258-9a55-421c-878d-73e0cc38ad4b 2024-09-17T23:48:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:48:16.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:48:16.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-17T23:48:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd1 2024-09-17T23:48:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:48:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd1 2024-09-17T23:48:17.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:48:17.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:48:17.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:48:17.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 0b23d82f-7087-49e5-ad42-c1529c1f693d 2024-09-17T23:48:17.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:48:17.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:48:17.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:17.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:48:17.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:48:17.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:48:17.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:48:17.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary2 2024-09-17T23:48:17.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd1 test-primary2 2024-09-17T23:48:17.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:48:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary2 2024-09-17T23:48:17.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:48:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary2 2024-09-17T23:48:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:48:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:48:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:48:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary2 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary3 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary3 snapshot 10G 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary2 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary2 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:17.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary3 10G 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary3 2024-09-17T23:48:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:48:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:48:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary3 snapshot 2024-09-17T23:48:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:17.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:17.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:17.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:17.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary3 snapshot 2024-09-17T23:48:17.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:48:17.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary2/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary2' 2024-09-17T23:48:17.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 6 2024-09-17T23:48:17.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:48:17.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary3 2024-09-17T23:48:17.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary3': 2024-09-17T23:48:17.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:48:17.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:48:17.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:48:17.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 10cccaddc562 2024-09-17T23:48:17.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.10cccaddc562 2024-09-17T23:48:17.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:48:17.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:48:17.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:48:17.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:48:17.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:48:17 2024 2024-09-17T23:48:17.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:48:17 2024 2024-09-17T23:48:17.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:48:17 2024 2024-09-17T23:48:17.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:48:17.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:48:17.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 3643a8f6-f271-4297-95a3-71f0728fb1fa 2024-09-17T23:48:17.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:48:17.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:48:17.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-17T23:48:18.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd2 2024-09-17T23:48:18.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:48:18.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd2 2024-09-17T23:48:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:48:18.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:48:18.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:48:18.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 13725491-4425-469a-a706-e4b011c0570d 2024-09-17T23:48:18.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:48:18.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:48:18.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:18.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:48:18.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:48:18.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:48:18.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:48:18.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:18.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary3 2024-09-17T23:48:18.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd2 test-primary3 2024-09-17T23:48:18.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:48:18.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary3 2024-09-17T23:48:18.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:48:18.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary3 2024-09-17T23:48:18.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:48:18.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary4 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary4 snapshot 10G 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary3 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary4 10G 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary3 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:48:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:48:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary3 2024-09-17T23:48:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:48:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary4 2024-09-17T23:48:18.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:48:18.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:18.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:48:18.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:18.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary4 snapshot 2024-09-17T23:48:18.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:18.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:18.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:18.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:18.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary4 snapshot 2024-09-17T23:48:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:48:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary3/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary3' 2024-09-17T23:48:18.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 8 2024-09-17T23:48:18.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:48:18.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:18.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary4 2024-09-17T23:48:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary4': 2024-09-17T23:48:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:48:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:48:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:48:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 10dc826433f3 2024-09-17T23:48:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.10dc826433f3 2024-09-17T23:48:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:48:18.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:48:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:48:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:48:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:48:18 2024 2024-09-17T23:48:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:48:18 2024 2024-09-17T23:48:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:48:18 2024 2024-09-17T23:48:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:48:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:48:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 969f453f-b516-4208-ac2e-cf5b99921fb5 2024-09-17T23:48:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:48:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:48:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-17T23:48:19.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd3 2024-09-17T23:48:19.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:48:19.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd3 2024-09-17T23:48:19.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:48:19.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:48:19.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:48:19.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: aeed2f79-4ce8-407e-96b9-6b92b12b21b4 2024-09-17T23:48:19.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:48:19.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:48:19.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:19.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:48:19.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:48:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:48:19.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:48:19.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:19.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary4 2024-09-17T23:48:19.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd3 test-primary4 2024-09-17T23:48:19.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:48:19.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary4 2024-09-17T23:48:19.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:48:19.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary4 2024-09-17T23:48:19.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:48:19.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:48:19.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary4 2024-09-17T23:48:19.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:19.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary5 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary5 snapshot 10G 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary4 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary4 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary5 10G 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:48:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:48:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:48:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary5 2024-09-17T23:48:19.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:48:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:48:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary5 snapshot 2024-09-17T23:48:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary5 snapshot 2024-09-17T23:48:19.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:48:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary4/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary4' 2024-09-17T23:48:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:48:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:19.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:48:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 10 2024-09-17T23:48:19.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:48:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:19.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary5 2024-09-17T23:48:19.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary5': 2024-09-17T23:48:19.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:48:19.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:48:19.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:48:19.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 10ec7a2bc892 2024-09-17T23:48:19.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.10ec7a2bc892 2024-09-17T23:48:19.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:48:19.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:48:19.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:48:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:48:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:48:19 2024 2024-09-17T23:48:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:48:19 2024 2024-09-17T23:48:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:48:19 2024 2024-09-17T23:48:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:48:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:48:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 551a4170-af79-4f62-9088-7d79bb565dcf 2024-09-17T23:48:19.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:48:19.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:48:19.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-17T23:48:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd4 2024-09-17T23:48:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:48:20.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd4 2024-09-17T23:48:20.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:48:20.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:48:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:48:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 14b6851f-a8c9-4f94-987a-216688d8f1d5 2024-09-17T23:48:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:48:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:48:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:48:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:48:20.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:48:20.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:48:20.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:20.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary5 2024-09-17T23:48:20.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd4 test-primary5 2024-09-17T23:48:20.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:48:20.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary5 2024-09-17T23:48:20.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary5 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary5 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:48:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary5 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary5 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary6 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary6 snapshot 10G 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:20.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary6 10G 2024-09-17T23:48:20.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:20.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:20.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:20.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:20.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:20.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:20.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:48:20.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:48:20.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:48:20.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:20.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary6 2024-09-17T23:48:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:48:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:48:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary6 snapshot 2024-09-17T23:48:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:20.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary6 snapshot 2024-09-17T23:48:20.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:48:20.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary5/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary5' 2024-09-17T23:48:20.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:20.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:48:20.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:20.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:20.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:20.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:48:20.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 13 2024-09-17T23:48:20.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:48:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:20.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary6 2024-09-17T23:48:20.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary6': 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 10ff223255a2 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.10ff223255a2 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:48:20 2024 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:48:20 2024 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:48:20 2024 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:48:20.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:48:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 2a0b583c-8984-4fe1-a8b4-539244bbc29d 2024-09-17T23:48:20.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:48:20.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:48:20.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-17T23:48:21.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd5 2024-09-17T23:48:21.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:48:21.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd5 2024-09-17T23:48:21.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:48:21.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:48:21.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:48:21.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 5471f952-96ce-4f52-a542-8ca6bba5f6a1 2024-09-17T23:48:21.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:48:21.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:48:21.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:21.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:48:21.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:48:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:48:21.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:48:21.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:21.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary6 2024-09-17T23:48:21.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd5 test-primary6 2024-09-17T23:48:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:48:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary6 2024-09-17T23:48:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:48:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary6 2024-09-17T23:48:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:48:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary6 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary6 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary6 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary7 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary7 snapshot 10G 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:21.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary7 10G 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:48:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:21.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary7 2024-09-17T23:48:21.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:48:21.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:21.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:48:21.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:21.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary7 snapshot 2024-09-17T23:48:21.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:21.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:21.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:48:21.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:21.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary7 snapshot 2024-09-17T23:48:21.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:48:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary6/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary6' 2024-09-17T23:48:21.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 12 2024-09-17T23:48:21.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:48:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:48:21.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:21.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 16 2024-09-17T23:48:21.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:48:21.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:21.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary7 2024-09-17T23:48:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary7': 2024-09-17T23:48:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:48:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:48:22.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1112d9e14199 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1112d9e14199 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:48:21 2024 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:48:21 2024 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:48:21 2024 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: a1cbc563-970b-461d-87b5-fa24b669fb62 2024-09-17T23:48:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:48:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:48:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-17T23:48:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd6 2024-09-17T23:48:22.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:48:22.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd6 2024-09-17T23:48:22.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:48:22.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:48:22.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:48:22.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 4b87f3d0-6df2-49a8-b6db-77d2e05b7562 2024-09-17T23:48:22.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:48:22.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:48:22.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:22.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:48:22.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:48:22.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:48:22.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:48:22.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary7 2024-09-17T23:48:22.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd6 test-primary7 2024-09-17T23:48:22.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:48:22.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary7 2024-09-17T23:48:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:48:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary7 2024-09-17T23:48:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:48:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:48:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary8 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary8 snapshot 10G 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary8 10G 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:22.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary8 2024-09-17T23:48:22.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary7 2024-09-17T23:48:22.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary7 2024-09-17T23:48:22.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary7 2024-09-17T23:48:22.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:48:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:48:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary8 snapshot 2024-09-17T23:48:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:48:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary8 snapshot 2024-09-17T23:48:22.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:48:22.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary7/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary7' 2024-09-17T23:48:22.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:22.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:48:22.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:22.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:22.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:22.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:48:22.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 15 2024-09-17T23:48:22.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:22.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 19 2024-09-17T23:48:22.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:23.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:23.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:48:23.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:23.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:23.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:23.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:48:23.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 18 2024-09-17T23:48:23.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:48:23.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary8 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary8': 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 112479938ad2 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.112479938ad2 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:48:22 2024 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:48:22 2024 2024-09-17T23:48:24.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:48:22 2024 2024-09-17T23:48:24.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:48:24.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:48:24.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 6acb2ebc-f153-4559-98ce-7aeb7798bb9f 2024-09-17T23:48:24.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:48:24.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:48:24.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-17T23:48:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd7 2024-09-17T23:48:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:48:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd7 2024-09-17T23:48:24.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:48:24.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:48:24.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:48:24.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: f9141cce-f1a6-4713-8774-698c536485fc 2024-09-17T23:48:24.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:48:24.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:48:24.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:24.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:48:24.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:48:24.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:48:24.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:48:24.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:24.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary8 2024-09-17T23:48:24.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd7 test-primary8 2024-09-17T23:48:24.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:48:24.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary8 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary8 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary8 2024-09-17T23:48:24.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:48:24.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:48:24.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary8 2024-09-17T23:48:24.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary8 2024-09-17T23:48:24.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-17T23:48:24.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary9 2024-09-17T23:48:24.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary9 snapshot 10G 2024-09-17T23:48:24.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:24.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:24.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:24.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:24.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:48:24.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:24.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:24.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:48:24.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:24.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary9 10G 2024-09-17T23:48:24.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:24.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:24.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:48:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:48:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:48:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:48:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary9 2024-09-17T23:48:24.796 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-17T23:48:24.790+0000 7f52d7c6c640 -1 reset not still connected to 0x557844149e10 2024-09-17T23:48:24.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:48:24.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:48:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary9 snapshot 2024-09-17T23:48:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:48:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary9 snapshot 2024-09-17T23:48:24.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:48:24.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary8/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary8' 2024-09-17T23:48:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:48:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:24.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:48:24.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 21 2024-09-17T23:48:24.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:24.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 23 2024-09-17T23:48:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:48:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:48:24.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:25.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:25.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:48:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:48:25.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 22 2024-09-17T23:48:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:48:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:48:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:48:25.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:25.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:48:25.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary9 2024-09-17T23:48:26.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary9': 2024-09-17T23:48:26.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:48:26.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:48:26.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 113722ec4008 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.113722ec4008 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:48:24 2024 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:48:24 2024 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:48:24 2024 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 7cb937fc-9ee4-4d15-a2c2-54ef26b55f87 2024-09-17T23:48:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:48:26.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:48:26.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-17T23:48:26.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd8 2024-09-17T23:48:26.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:48:26.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd8 2024-09-17T23:48:26.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:48:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:48:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:48:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: b37d6b0b-4bca-40e2-b128-8759a90b4bfc 2024-09-17T23:48:26.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:48:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:48:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:48:26.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:48:26.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:48:26.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:48:26.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:26.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary9 2024-09-17T23:48:26.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd8 test-primary9 2024-09-17T23:48:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 26 2024-09-17T23:48:26.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 25 2024-09-17T23:48:26.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:26.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:26.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:48:26.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary9 2024-09-17T23:48:26.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:48:26.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary9 2024-09-17T23:48:26.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:26.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:26.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:48:26.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:48:26.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:26.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:48:26.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:48:26.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary10 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary10 snapshot 10G 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary9 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:48:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary9 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary9 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary10 10G 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:48:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:48:26.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:48:26.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:48:26.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary10 2024-09-17T23:48:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:48:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:26.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:26.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:26.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:48:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:48:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:48:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:48:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary10 snapshot 2024-09-17T23:48:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:48:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:48:26.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary10 snapshot 2024-09-17T23:48:27.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:48:27.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary9/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary9' 2024-09-17T23:48:27.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 27 2024-09-17T23:48:27.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 28 2024-09-17T23:48:27.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:27.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:27.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:48:27.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:27.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:27.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:27.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:48:27.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:48:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:48:27.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:48:27.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 29 2024-09-17T23:48:27.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:48:27.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:27.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary10 2024-09-17T23:48:28.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary10': 2024-09-17T23:48:28.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:48:28.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:48:28.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:48:28.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 115516e0b18d 2024-09-17T23:48:28.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.115516e0b18d 2024-09-17T23:48:28.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:48:28.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:48:28.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:48:28.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:48:28.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:48:26 2024 2024-09-17T23:48:28.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:48:26 2024 2024-09-17T23:48:28.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:48:26 2024 2024-09-17T23:48:28.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:48:28.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:48:28.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 7fdefe51-09c9-4400-ae06-6ebd475db1ce 2024-09-17T23:48:28.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:48:28.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:48:28.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-17T23:48:28.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd9 2024-09-17T23:48:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:48:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd9 2024-09-17T23:48:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:48:28.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:48:28.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:48:28.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: fb0a14a6-b5ba-4034-b19a-3327a891e30b 2024-09-17T23:48:28.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:48:28.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:48:28.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:28.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:48:28.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:48:28.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:48:28.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:48:28.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:48:28.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary10 2024-09-17T23:48:28.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd9 test-primary10 2024-09-17T23:48:28.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:48:28.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary10 2024-09-17T23:48:28.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:48:28.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary10 2024-09-17T23:48:28.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:28.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary10 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 30300 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary10 2024-09-17T23:48:28.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary10 2024-09-17T23:48:28.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:48:28.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary10/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary10' 2024-09-17T23:48:28.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 30 2024-09-17T23:48:28.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:28.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:28.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:48:28.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:28.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:28.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:28.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:48:28.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 34 2024-09-17T23:48:29.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:48:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:48:29.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:29.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:48:29.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:29.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:29.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:29.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:48:29.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 33 2024-09-17T23:48:29.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:29.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 32 2024-09-17T23:48:29.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:30.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 35 2024-09-17T23:48:30.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:30.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:30.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:48:30.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:30.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:30.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:30.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:48:30.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:30.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:48:30.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:30.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:30.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:48:30.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:48:30.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:48:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:48:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:48:31.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 36 2024-09-17T23:48:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 37 2024-09-17T23:48:31.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:31.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:31.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:31.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:48:31.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:31.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:31.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:31.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:48:32.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:32.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:48:32.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:32.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:32.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:48:32.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:48:32.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:32.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:48:32.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:32.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:32.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:48:32.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:48:32.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 38 2024-09-17T23:48:32.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:32.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:48:32.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:32.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:32.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:32.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:48:32.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 39 2024-09-17T23:48:33.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 40 2024-09-17T23:48:33.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:33.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:48:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:48:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 42 2024-09-17T23:48:34.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 41 2024-09-17T23:48:34.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:34.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:34.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:34.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:48:34.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:34.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:34.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:34.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:48:34.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:34.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:48:34.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:34.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:34.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:34.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:48:34.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 43 2024-09-17T23:48:34.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 44 2024-09-17T23:48:35.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:35.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 45 2024-09-17T23:48:35.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:36.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:36.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:48:36.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:36.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:36.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:36.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:48:36.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:36.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:48:36.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:36.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:36.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:48:36.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:48:36.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:36.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:48:36.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:36.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:36.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:48:36.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:48:36.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 47 2024-09-17T23:48:36.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 46 2024-09-17T23:48:36.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:37.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:37.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:48:37.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:37.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:37.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:48:37.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:48:37.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:37.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:48:37.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:37.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:37.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:37.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:48:37.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:37.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:48:37.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:37.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:37.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:48:37.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:48:38.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:38.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:48:38.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:38.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:38.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:38.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:48:38.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 50 2024-09-17T23:48:38.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 49 2024-09-17T23:48:38.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 48 2024-09-17T23:48:38.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:38.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:38.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:38.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:38.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:48:38.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:38.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:38.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:38.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:48:39.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 52 2024-09-17T23:48:39.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 51 2024-09-17T23:48:39.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:39.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:39.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:39.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:48:39.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:39.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:39.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:39.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:48:39.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:39.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:48:39.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:39.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:39.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:39.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:48:40.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 53 2024-09-17T23:48:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 54 2024-09-17T23:48:40.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:40.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:40.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 55 2024-09-17T23:48:41.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:48:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:48:41.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:48:41.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:41.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:48:41.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:41.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:41.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:41.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:48:41.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:41.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:48:41.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:41.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:41.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:48:41.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:48:41.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 56 2024-09-17T23:48:41.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 57 2024-09-17T23:48:41.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:42.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:42.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:48:42.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:42.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:42.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:42.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:48:42.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:42.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:48:42.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:42.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:42.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:48:42.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:48:42.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 60 2024-09-17T23:48:42.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 58 2024-09-17T23:48:42.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:42.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:42.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 59 2024-09-17T23:48:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:43.038 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-17T23:48:43.034+0000 7f52d7c6c640 -1 reset not still connected to 0x5578449cd2b0 2024-09-17T23:48:43.038 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-17T23:48:43.034+0000 7f52d7c6c640 -1 reset not still connected to 0x55784508c000 2024-09-17T23:48:43.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:43.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:48:43.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:43.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:43.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:43.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:48:43.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:48:43.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 61 2024-09-17T23:48:43.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 62 2024-09-17T23:48:43.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:43.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:44.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:44.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:48:44.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:44.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:44.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:44.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:48:44.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 63 2024-09-17T23:48:44.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 64 2024-09-17T23:48:44.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:44.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:44.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:44.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:48:44.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:44.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:44.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:44.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:48:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:48:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:48:45.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 65 2024-09-17T23:48:45.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:45.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:45.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:48:45.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:45.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:45.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:45.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:48:45.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:45.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:48:45.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:45.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:45.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:48:45.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:48:45.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:45.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:48:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:48:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:48:46.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 67 2024-09-17T23:48:46.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 66 2024-09-17T23:48:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:46.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:46.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:46.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:48:46.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:46.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:46.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:48:46.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:48:46.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:46.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:48:46.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:46.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:46.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:46.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:48:47.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 68 2024-09-17T23:48:47.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:47.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:48:47.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:47.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:47.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:47.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:48:47.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:47.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:48:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:48:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:48:47.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 69 2024-09-17T23:48:47.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 70 2024-09-17T23:48:47.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:47.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:48:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:48.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:48:48.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 72 2024-09-17T23:48:48.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 71 2024-09-17T23:48:48.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:48.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:49.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 74 2024-09-17T23:48:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:48:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:48:49.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:49.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:48:49.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:49.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:49.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:49.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:48:49.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 73 2024-09-17T23:48:49.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:49.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:50.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:50.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:48:50.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:50.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:50.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:50.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:48:50.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 75 2024-09-17T23:48:50.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:50.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:50.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:48:50.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:50.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:50.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:48:50.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:48:50.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:50.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:48:50.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:50.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:50.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:48:50.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:48:51.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 76 2024-09-17T23:48:51.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 77 2024-09-17T23:48:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:51.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:51.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:51.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:48:51.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:51.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:51.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:51.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:48:51.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:48:52.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 80 2024-09-17T23:48:52.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 79 2024-09-17T23:48:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 78 2024-09-17T23:48:52.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:52.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:52.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:52.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:52.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:48:52.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:52.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:52.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:48:52.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:48:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:48:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:48:53.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:53.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:48:53.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:53.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:53.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:53.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:48:53.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 82 2024-09-17T23:48:53.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 81 2024-09-17T23:48:53.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:53.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:48:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:54.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:48:54.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:54.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:48:54.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:54.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:54.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:54.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:48:55.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 84 2024-09-17T23:48:55.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 83 2024-09-17T23:48:55.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:55.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:55.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:55.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:48:55.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:55.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:55.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:48:55.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:48:55.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:55.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:48:55.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:55.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:55.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:48:55.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:48:55.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:55.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:48:55.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:55.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:55.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:48:55.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:48:56.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 85 2024-09-17T23:48:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:56.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 87 2024-09-17T23:48:56.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 86 2024-09-17T23:48:56.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:56.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:48:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:48:56.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:48:56.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:56.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:48:56.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:56.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:56.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:48:56.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:48:57.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 89 2024-09-17T23:48:57.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:58.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 90 2024-09-17T23:48:58.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 88 2024-09-17T23:48:58.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:58.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:58.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:48:58.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:58.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:58.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:48:58.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:48:58.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:58.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:48:58.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:58.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:58.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:48:58.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:48:58.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:59.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:59.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:48:59.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:59.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:59.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:48:59.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:48:59.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 92 2024-09-17T23:48:59.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 91 2024-09-17T23:48:59.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:59.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:48:59.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:59.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:48:59.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:59.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:59.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:48:59.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:48:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:48:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:48:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:48:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:48:59.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:48:59.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:49:00.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 93 2024-09-17T23:49:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 94 2024-09-17T23:49:00.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:00.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:00.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:00.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:49:00.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:00.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:00.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:49:00.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:49:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 95 2024-09-17T23:49:01.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:01.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:01.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:49:01.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:01.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:01.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:49:01.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:49:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:49:01.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:01.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:01.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:49:01.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:49:01.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 97 2024-09-17T23:49:01.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 96 2024-09-17T23:49:01.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:01.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:49:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:49:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:49:02.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:02.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:49:02.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:02.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:02.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:49:02.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:49:02.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 100 2024-09-17T23:49:02.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 99 2024-09-17T23:49:02.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 98 2024-09-17T23:49:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:02.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:02.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:03.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:03.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:49:03.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:03.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:03.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:49:03.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:49:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:49:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:49:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:49:03.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 102 2024-09-17T23:49:03.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 101 2024-09-17T23:49:03.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:03.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:04.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:04.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:49:04.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:04.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:04.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:49:04.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:49:04.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 103 2024-09-17T23:49:04.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 104 2024-09-17T23:49:04.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:04.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:49:04.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:04.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:04.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:49:04.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:49:04.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:04.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:49:04.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:04.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:04.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:49:04.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:49:04.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:04.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:05.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 105 2024-09-17T23:49:05.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:05.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:05.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:49:05.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:05.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:05.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:49:05.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:49:05.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:05.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:49:05.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:49:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:49:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:49:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:49:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:49:06.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 106 2024-09-17T23:49:06.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 107 2024-09-17T23:49:06.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:06.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:49:06.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:06.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:06.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:49:06.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:49:06.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:06.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:06.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:06.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:49:06.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:06.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:06.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:49:06.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:49:07.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 108 2024-09-17T23:49:07.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:07.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:49:07.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:07.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:07.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:49:07.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:49:07.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:07.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:49:07.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:07.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:07.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:49:07.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:49:07.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:07.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 110 2024-09-17T23:49:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 109 2024-09-17T23:49:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:07.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:08.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 112 2024-09-17T23:49:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 111 2024-09-17T23:49:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:49:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:08.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:49:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:49:08.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:08.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:09.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:09.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:49:09.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:09.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:09.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:49:09.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:49:09.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:09.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:49:09.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:09.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:09.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:49:09.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:49:09.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 114 2024-09-17T23:49:09.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 113 2024-09-17T23:49:09.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:10.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:10.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:49:10.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:10.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:10.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:49:10.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:49:10.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 115 2024-09-17T23:49:10.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:10.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:49:10.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:10.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:10.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:49:10.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:49:10.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:10.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:49:10.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:10.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:10.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:49:10.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:49:10.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:11.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 117 2024-09-17T23:49:11.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:11.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 116 2024-09-17T23:49:11.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:11.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:11.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:49:11.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:11.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:11.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:49:11.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:49:11.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:11.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:49:11.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:11.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:11.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:49:11.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:49:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 120 2024-09-17T23:49:12.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:12.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 118 2024-09-17T23:49:12.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:12.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:49:12.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:12.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:12.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:49:12.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:49:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 119 2024-09-17T23:49:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:49:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:49:12.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:49:12.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:12.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:13.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:13.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:49:13.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:13.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:13.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:49:13.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:49:13.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 121 2024-09-17T23:49:13.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 122 2024-09-17T23:49:13.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:13.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:49:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:14.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:49:14.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:49:14.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:14.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:49:14.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:14.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:14.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:49:14.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:49:14.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 124 2024-09-17T23:49:14.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 123 2024-09-17T23:49:14.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:14.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:49:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:49:15.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:49:15.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 125 2024-09-17T23:49:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:49:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:49:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:49:15.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:15.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:15.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:49:15.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:15.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:15.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:49:15.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:49:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 126 2024-09-17T23:49:16.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 127 2024-09-17T23:49:16.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:16.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:16.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:16.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:49:16.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:16.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:16.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:49:16.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:49:16.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:16.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:49:16.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:16.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:16.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:49:16.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:49:17.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 128 2024-09-17T23:49:17.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:17.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:49:17.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:17.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:17.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:49:17.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:49:17.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 130 2024-09-17T23:49:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:49:17.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:49:17.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:49:18.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 129 2024-09-17T23:49:18.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:18.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:18.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:18.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:49:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:49:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:49:18.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 132 2024-09-17T23:49:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 131 2024-09-17T23:49:18.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:19.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:19.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:19.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:49:19.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:19.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:19.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:49:19.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:49:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:49:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:49:19.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:49:19.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 133 2024-09-17T23:49:20.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:20.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 134 2024-09-17T23:49:20.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:20.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 135 2024-09-17T23:49:21.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:21.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:21.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:21.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:49:21.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:49:21.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:21.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:21.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:21.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:49:21.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:21.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:49:21.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:49:21.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:49:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:49:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:21.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:21.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:49:21.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:49:21.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 137 2024-09-17T23:49:21.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 136 2024-09-17T23:49:21.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:21.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:21.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:49:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:49:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:49:22.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:22.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:49:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:49:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:49:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:49:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:49:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:49:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 139 2024-09-17T23:49:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 138 2024-09-17T23:49:23.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:23.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:49:23.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:23.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:23.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:49:23.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:49:23.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 140 2024-09-17T23:49:23.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:23.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:23.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:24.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 142 2024-09-17T23:49:24.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:24.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:49:24.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:24.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:24.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:49:24.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:49:24.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 141 2024-09-17T23:49:24.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:49:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:24.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:24.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:49:24.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:49:24.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:24.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:49:24.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:24.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:24.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:49:24.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:49:25.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 144 2024-09-17T23:49:25.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 143 2024-09-17T23:49:25.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:25.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:26.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 145 2024-09-17T23:49:26.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:49:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:49:26.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:49:26.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:26.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:49:26.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:26.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:26.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:49:26.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:49:26.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:26.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:49:26.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:26.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:26.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:49:26.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:49:26.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 147 2024-09-17T23:49:26.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 146 2024-09-17T23:49:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:26.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:27.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:27.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:49:27.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:27.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:27.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:49:27.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:49:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:49:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:49:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:49:27.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 148 2024-09-17T23:49:27.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 150 2024-09-17T23:49:27.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:27.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:27.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 149 2024-09-17T23:49:27.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:28.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:28.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:49:28.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:28.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:28.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:49:28.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:49:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:49:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:49:28.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:49:29.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:29.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:49:29.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:29.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:29.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:49:29.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:49:29.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 151 2024-09-17T23:49:29.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 152 2024-09-17T23:49:29.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:29.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:29.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:49:29.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:29.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:29.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:49:29.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:49:29.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:29.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:49:29.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:29.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:29.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:49:29.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:49:30.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 153 2024-09-17T23:49:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 154 2024-09-17T23:49:30.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:30.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:49:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:49:30.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:49:30.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:30.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:49:30.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:30.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:30.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:49:30.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:49:30.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:49:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:49:30.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:49:31.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 155 2024-09-17T23:49:31.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 157 2024-09-17T23:49:31.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 156 2024-09-17T23:49:31.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:31.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:32.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:32.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:49:32.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:32.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:32.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:49:32.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:49:32.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:32.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:49:32.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:32.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:32.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:49:32.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:49:32.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 158 2024-09-17T23:49:32.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:32.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 159 2024-09-17T23:49:32.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 160 2024-09-17T23:49:32.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:32.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:49:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:49:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:49:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:49:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:49:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:49:33.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 162 2024-09-17T23:49:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 161 2024-09-17T23:49:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:33.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:34.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:34.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:49:34.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:34.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:34.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:49:34.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:49:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 164 2024-09-17T23:49:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:49:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:49:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:49:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:49:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:49:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:49:34.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 163 2024-09-17T23:49:34.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:34.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:35.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 165 2024-09-17T23:49:35.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:35.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:49:35.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:35.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:35.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:49:35.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:49:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:49:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:49:35.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:49:35.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:35.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:49:35.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:35.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:35.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:49:35.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:49:36.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 166 2024-09-17T23:49:36.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 167 2024-09-17T23:49:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:49:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:49:36.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:49:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:49:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:36.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:36.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:49:36.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:49:37.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 168 2024-09-17T23:49:37.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 169 2024-09-17T23:49:37.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 170 2024-09-17T23:49:37.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:37.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:49:37.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:37.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:37.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:49:37.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:49:37.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:37.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:49:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:49:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:49:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:49:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:49:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:49:38.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 171 2024-09-17T23:49:38.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 172 2024-09-17T23:49:38.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:38.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:39.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:39.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:49:39.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:39.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:39.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:39.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:49:39.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:49:39.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:39.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:39.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:49:39.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:49:39.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:49:39.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 174 2024-09-17T23:49:39.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 173 2024-09-17T23:49:39.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:39.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:40.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 175 2024-09-17T23:49:40.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:40.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:40.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:49:40.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:40.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:40.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:49:40.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:40.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:49:40.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:49:40.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:40.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:40.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:49:40.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:49:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:49:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:49:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:49:41.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 176 2024-09-17T23:49:41.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 177 2024-09-17T23:49:41.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:41.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:41.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:41.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:49:41.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:41.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:41.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:49:41.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:49:41.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:41.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:49:41.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:41.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:41.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:49:41.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:49:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 180 2024-09-17T23:49:42.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:42.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:49:42.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:42.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:42.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:49:42.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:49:42.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:42.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:49:42.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:42.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:42.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:49:42.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:49:42.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:42.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 179 2024-09-17T23:49:42.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 178 2024-09-17T23:49:42.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:49:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:49:43.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:49:43.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 182 2024-09-17T23:49:43.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 181 2024-09-17T23:49:43.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:43.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:49:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:44.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:44.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:49:44.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:49:44.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:44.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:49:44.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:44.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:44.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:49:44.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:49:44.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 184 2024-09-17T23:49:44.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 183 2024-09-17T23:49:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:45.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 185 2024-09-17T23:49:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:49:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:49:45.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:49:45.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:45.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:45.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:49:45.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:45.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:45.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:49:45.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:49:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:49:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:49:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:49:46.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 187 2024-09-17T23:49:46.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 186 2024-09-17T23:49:46.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:46.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:49:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:49:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:49:46.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:46.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:49:46.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:46.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:46.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:49:46.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:49:47.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 188 2024-09-17T23:49:47.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:47.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:49:47.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:47.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:47.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:49:47.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:49:47.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:47.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:49:47.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:47.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:47.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:49:47.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:49:47.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 190 2024-09-17T23:49:47.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:48.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:48.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 189 2024-09-17T23:49:48.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:49:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:49:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:49:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 192 2024-09-17T23:49:48.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 191 2024-09-17T23:49:49.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:49:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:49:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:49:49.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:49.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:49:49.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:49.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:49.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:49:49.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:49:50.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 194 2024-09-17T23:49:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 193 2024-09-17T23:49:50.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:50.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:50.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:50.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:49:50.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:50.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:50.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:49:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:49:51.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 195 2024-09-17T23:49:51.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:51.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:49:51.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:51.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:51.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:49:51.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:49:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:49:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:49:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:49:51.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 196 2024-09-17T23:49:51.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 197 2024-09-17T23:49:51.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:51.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:52.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:52.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:49:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:49:52.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:49:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:49:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:49:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:49:53.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 198 2024-09-17T23:49:53.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:53.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:49:53.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:53.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:53.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:49:53.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:49:53.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:53.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:49:53.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:53.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:53.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:49:53.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:49:53.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 200 2024-09-17T23:49:53.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 199 2024-09-17T23:49:53.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:53.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:53.667 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-17T23:49:53.660+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-17T23:49:53.669 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-17T23:49:53.664+0000 7f52d7c6c640 -1 reset not still connected to 0x557844149e10 2024-09-17T23:49:53.670 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-17T23:49:53.664+0000 7f52d7c6c640 -1 reset not still connected to 0x5578449cd2b0 2024-09-17T23:49:53.670 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-17T23:49:53.664+0000 7f52d7c6c640 -1 reset not still connected to 0x55784508c000 2024-09-17T23:49:54.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:54.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:49:54.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:54.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:54.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:49:54.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:49:54.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 201 2024-09-17T23:49:54.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 202 2024-09-17T23:49:54.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:54.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:54.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:54.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:49:54.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:54.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:54.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:49:54.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:49:54.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:54.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:49:54.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:54.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:54.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:49:54.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:49:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 204 2024-09-17T23:49:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 203 2024-09-17T23:49:55.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 205 2024-09-17T23:49:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:49:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:49:56.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:49:56.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:56.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:56.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:49:56.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:56.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:56.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:49:56.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:49:56.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:56.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:49:56.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:56.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:56.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:49:56.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:49:56.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 207 2024-09-17T23:49:56.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 206 2024-09-17T23:49:56.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:56.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:57.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:57.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:49:57.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:57.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:57.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:49:57.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:49:57.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:57.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:49:57.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:57.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:57.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:49:57.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:49:57.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 208 2024-09-17T23:49:57.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 210 2024-09-17T23:49:57.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:57.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 209 2024-09-17T23:49:57.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:57.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:58.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:58.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:49:58.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:58.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:58.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:49:58.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:49:58.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:58.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:49:58.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:58.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:58.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:49:58.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:49:59.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:59.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:49:59.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:59.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:59.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:49:59.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:49:59.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 212 2024-09-17T23:49:59.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 211 2024-09-17T23:49:59.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:59.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:49:59.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:49:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:49:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:49:59.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:49:59.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:49:59.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:49:59.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:49:59.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:49:59.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:50:00.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 213 2024-09-17T23:50:00.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 214 2024-09-17T23:50:00.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:00.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:00.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:50:00.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:00.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:00.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:50:00.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:50:00.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:00.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:50:00.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:00.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:00.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:50:00.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:50:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:50:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:00.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:50:00.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:50:01.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 215 2024-09-17T23:50:01.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:01.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 217 2024-09-17T23:50:01.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 216 2024-09-17T23:50:01.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:01.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:50:02.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:50:02.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 218 2024-09-17T23:50:02.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 219 2024-09-17T23:50:02.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 220 2024-09-17T23:50:02.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:02.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:02.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:03.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:03.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:50:03.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:03.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:03.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:50:03.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:50:03.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:03.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:50:03.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:03.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:03.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:50:03.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:50:03.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 222 2024-09-17T23:50:03.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 221 2024-09-17T23:50:03.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:03.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:04.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:04.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:50:04.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:04.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:04.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:50:04.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:50:04.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 223 2024-09-17T23:50:04.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:04.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 224 2024-09-17T23:50:04.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:04.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:50:04.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:04.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:04.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:50:04.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:50:04.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:04.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:04.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:50:04.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:04.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:04.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:50:04.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:50:05.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 225 2024-09-17T23:50:05.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:05.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:05.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:50:05.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:05.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:05.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:50:05.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:50:05.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:05.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:50:05.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:50:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:50:05.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:05.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:50:05.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:05.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:05.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:50:05.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:50:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 226 2024-09-17T23:50:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 227 2024-09-17T23:50:06.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:06.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:50:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:06.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:50:06.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:50:06.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:06.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:50:06.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:06.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:06.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:50:06.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:50:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:50:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:50:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:50:07.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 228 2024-09-17T23:50:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:50:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:07.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:50:07.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:50:07.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 230 2024-09-17T23:50:07.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:07.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 229 2024-09-17T23:50:07.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:07.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:08.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:08.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:50:08.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:08.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:08.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:50:08.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:50:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 232 2024-09-17T23:50:08.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 231 2024-09-17T23:50:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:08.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:09.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:09.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:50:09.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:09.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:09.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:50:09.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:50:09.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:09.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:50:09.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:09.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:09.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:50:09.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:50:09.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 234 2024-09-17T23:50:09.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 233 2024-09-17T23:50:09.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:09.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:10.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 235 2024-09-17T23:50:10.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:10.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:50:10.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:10.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:10.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:50:10.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:50:10.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:10.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:50:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:50:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:50:10.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:10.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:50:10.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:10.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:10.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:50:10.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:50:11.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 236 2024-09-17T23:50:11.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 237 2024-09-17T23:50:11.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:11.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:11.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:50:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:50:11.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:50:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:50:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:50:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:50:12.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:12.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:50:12.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:12.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:12.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:50:12.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:50:12.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:50:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:50:12.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:50:12.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 239 2024-09-17T23:50:12.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 240 2024-09-17T23:50:12.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 238 2024-09-17T23:50:13.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:13.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:13.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:50:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:50:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:50:13.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 242 2024-09-17T23:50:14.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 241 2024-09-17T23:50:14.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:14.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:50:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:50:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:50:14.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:14.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:50:14.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:14.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:14.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:50:14.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:50:14.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 244 2024-09-17T23:50:14.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:14.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 243 2024-09-17T23:50:15.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:15.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 245 2024-09-17T23:50:16.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:16.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:50:16.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:50:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:50:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:50:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:50:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:50:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:50:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:50:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:50:16.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:16.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 246 2024-09-17T23:50:16.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 247 2024-09-17T23:50:16.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:16.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:17.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:17.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:50:17.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:17.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:17.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:50:17.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:50:17.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:50:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:50:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:50:17.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 250 2024-09-17T23:50:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:50:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:50:17.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:50:18.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:18.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:50:18.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:18.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:18.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:50:18.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:50:18.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 248 2024-09-17T23:50:18.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:18.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 249 2024-09-17T23:50:18.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:50:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:50:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:50:19.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 252 2024-09-17T23:50:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 251 2024-09-17T23:50:19.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:19.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:50:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:19.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:50:19.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:50:19.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:50:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 253 2024-09-17T23:50:20.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:20.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 254 2024-09-17T23:50:20.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:50:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:50:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:50:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 255 2024-09-17T23:50:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:50:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:50:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:50:21.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:21.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:21.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:50:21.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:21.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:21.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:50:21.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:50:21.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 257 2024-09-17T23:50:21.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 256 2024-09-17T23:50:21.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:21.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:22.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:22.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:50:22.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:22.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:22.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:50:22.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:50:22.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:22.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:50:22.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:22.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:22.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:50:22.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:50:23.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:23.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:50:23.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:23.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:23.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:50:23.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:50:23.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 258 2024-09-17T23:50:23.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:23.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:50:23.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:23.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:23.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:50:23.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:50:23.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 260 2024-09-17T23:50:23.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 259 2024-09-17T23:50:23.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:24.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:24.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:50:24.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:24.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:24.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:50:24.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:50:24.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 262 2024-09-17T23:50:24.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 261 2024-09-17T23:50:24.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:24.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:24.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:24.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:50:24.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:24.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:24.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:50:24.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:50:24.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:24.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:50:24.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:24.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:24.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:50:24.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:50:25.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 263 2024-09-17T23:50:25.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 264 2024-09-17T23:50:25.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:25.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:26.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 265 2024-09-17T23:50:26.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:26.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:26.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:50:26.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:26.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:26.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:50:26.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:50:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:50:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:50:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:50:26.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:26.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:50:26.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:26.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:26.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:50:26.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:50:26.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 266 2024-09-17T23:50:26.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 267 2024-09-17T23:50:26.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:26.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:27.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:27.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:50:27.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:27.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:27.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:50:27.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:50:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:50:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:50:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:50:27.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 269 2024-09-17T23:50:27.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 268 2024-09-17T23:50:27.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 270 2024-09-17T23:50:27.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:27.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:27.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:28.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:28.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:50:28.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:28.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:28.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:50:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:50:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:50:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:50:28.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:50:28.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 272 2024-09-17T23:50:28.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 271 2024-09-17T23:50:28.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:28.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:29.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:29.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:50:29.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:29.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:29.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:50:29.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:50:29.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 274 2024-09-17T23:50:29.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 273 2024-09-17T23:50:29.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:29.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:29.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:29.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:50:29.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:29.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:29.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:50:29.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:50:29.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:29.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:50:29.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:29.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:29.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:50:29.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:50:30.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 275 2024-09-17T23:50:30.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:30.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:50:30.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:30.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:30.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:50:30.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:50:30.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:30.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:50:30.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:30.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:30.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:50:30.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:50:30.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:30.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:50:30.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:30.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:30.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:50:30.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:50:30.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:31.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 276 2024-09-17T23:50:31.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 277 2024-09-17T23:50:31.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:31.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:31.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:31.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:50:31.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:31.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:31.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:50:31.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:50:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:50:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:50:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:50:32.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:32.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:50:32.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:32.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:32.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:50:32.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:50:32.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:50:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:50:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:50:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 279 2024-09-17T23:50:32.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:32.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 278 2024-09-17T23:50:32.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 280 2024-09-17T23:50:32.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:32.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:33.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:33.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:50:33.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:33.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:33.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:50:33.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:50:33.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 282 2024-09-17T23:50:33.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 281 2024-09-17T23:50:33.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:33.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:50:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:50:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:50:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:50:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:50:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:50:34.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 283 2024-09-17T23:50:34.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 284 2024-09-17T23:50:34.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:35.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:35.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:50:35.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:35.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:35.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:50:35.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:50:35.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 285 2024-09-17T23:50:35.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:35.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:35.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:50:35.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:35.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:35.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:50:35.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:50:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:50:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:50:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:50:36.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 286 2024-09-17T23:50:36.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 287 2024-09-17T23:50:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:36.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:50:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:50:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:50:36.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:36.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:50:36.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:36.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:36.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:50:36.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:50:37.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:37.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:50:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:50:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 288 2024-09-17T23:50:37.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:37.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:50:37.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:37.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:37.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:50:37.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:50:37.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 289 2024-09-17T23:50:38.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 290 2024-09-17T23:50:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:38.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:38.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:38.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:38.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:50:38.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:38.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:38.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:50:38.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:50:38.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 292 2024-09-17T23:50:38.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 291 2024-09-17T23:50:39.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:39.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:39.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:39.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:50:39.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:39.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:39.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:50:39.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:50:39.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:39.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:50:39.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:39.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:39.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:50:39.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:50:40.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 294 2024-09-17T23:50:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 293 2024-09-17T23:50:40.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:40.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 295 2024-09-17T23:50:40.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:50:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:50:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:50:41.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:41.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:50:41.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:41.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:41.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:50:41.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:50:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:50:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:41.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:50:41.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:50:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 297 2024-09-17T23:50:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 296 2024-09-17T23:50:41.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:41.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:42.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:42.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:50:42.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:42.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:42.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:50:42.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:50:42.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:42.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:50:42.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:42.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:42.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:50:42.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:50:42.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 299 2024-09-17T23:50:42.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 300 2024-09-17T23:50:43.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:43.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 298 2024-09-17T23:50:43.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:43.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:43.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:43.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:50:43.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:43.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:43.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:50:43.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:50:43.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:43.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:50:43.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:43.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:43.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:50:43.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:50:43.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:43.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:50:43.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:43.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:43.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:50:43.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:50:44.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 302 2024-09-17T23:50:44.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:44.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 301 2024-09-17T23:50:44.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:44.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:44.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:50:44.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:44.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:44.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:50:44.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:50:44.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:50:44.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 30436 2024-09-17T23:50:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 304 2024-09-17T23:50:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 303 2024-09-17T23:50:45.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:50:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 30578 2024-09-17T23:50:46.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:46.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:50:46.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:46.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:46.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:50:46.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:50:46.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:46.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:50:46.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:46.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:46.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:50:46.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:50:46.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 305 2024-09-17T23:50:46.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:46.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 306 2024-09-17T23:50:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:47.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:47.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:50:47.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:47.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:47.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:50:47.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:50:47.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:50:47.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 30712 2024-09-17T23:50:48.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:48.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:50:48.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:48.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:48.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:50:48.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:50:48.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 307 2024-09-17T23:50:48.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 308 2024-09-17T23:50:48.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:50:48.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 30867 2024-09-17T23:50:48.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:48.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:49.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:50:49.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 31021 2024-09-17T23:50:49.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 309 2024-09-17T23:50:49.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:50:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 31177 2024-09-17T23:50:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 310 2024-09-17T23:50:50.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:50:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:50:51.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:50:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:50:51.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 31353 2024-09-17T23:50:52.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:50:52.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:50:52.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:50:52.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:50:52.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:50:52.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:50:52.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 311 2024-09-17T23:50:52.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:53.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:50:53.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 31599 2024-09-17T23:50:53.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 312 2024-09-17T23:50:53.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:50:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:50:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 31795 2024-09-17T23:50:56.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-17T23:50:56.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-17T23:50:56.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 30301 2024-09-17T23:53:16.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-17T23:53:16.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-17T23:53:16.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-17T23:53:16.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 30437 2024-09-17T23:53:17.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-17T23:53:17.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-17T23:53:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-17T23:53:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 30579 2024-09-17T23:53:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-17T23:53:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-17T23:53:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-17T23:53:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 30713 2024-09-17T23:53:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-17T23:53:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-17T23:53:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-17T23:53:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 30868 2024-09-17T23:53:20.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-17T23:53:20.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-17T23:53:20.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-17T23:53:20.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 31022 2024-09-17T23:53:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-17T23:53:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-17T23:53:21.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-17T23:53:21.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 31178 2024-09-17T23:53:22.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-17T23:53:22.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-17T23:53:22.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-17T23:53:22.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 31354 2024-09-17T23:53:24.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-17T23:53:24.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-17T23:53:24.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-17T23:53:24.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 31600 2024-09-17T23:53:27.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-17T23:53:27.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-17T23:53:27.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-17T23:53:27.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 31796 2024-09-17T23:53:28.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-17T23:53:28.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-17T23:53:28.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-17T23:53:28.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:53:28.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-17T23:53:28.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:53:28.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 1 2024-09-17T23:53:28.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-17T23:53:28.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd0 2024-09-17T23:53:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:53:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary1 2024-09-17T23:53:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 2 2024-09-17T23:53:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary1 2024-09-17T23:53:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-17T23:53:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary1 2024-09-17T23:53:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd1 2024-09-17T23:53:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-17T23:53:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary2 2024-09-17T23:53:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary2 2024-09-17T23:53:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-17T23:53:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary2 2024-09-17T23:53:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:53:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 3 2024-09-17T23:53:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd2 2024-09-17T23:53:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-17T23:53:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary3 2024-09-17T23:53:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary3 2024-09-17T23:53:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-17T23:53:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:53:28.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 4 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary3 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd3 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary4 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary4 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary4 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 5 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd4 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary5 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary5 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-17T23:53:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary5 2024-09-17T23:53:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-17T23:53:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:53:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 6 2024-09-17T23:53:28.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd5 2024-09-17T23:53:28.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 7 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary6 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary6 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary6 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd6 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary7 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary7 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary7 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-17T23:53:28.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:53:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-17T23:53:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:53:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 8 2024-09-17T23:53:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd7 2024-09-17T23:53:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary8 2024-09-17T23:53:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-17T23:53:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary8 2024-09-17T23:53:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-17T23:53:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary8 2024-09-17T23:53:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-17T23:53:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 37727 2024-09-17T23:53:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 10 2024-09-17T23:53:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd9 2024-09-17T23:53:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary10 2024-09-17T23:53:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary10 2024-09-17T23:53:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-17T23:53:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary10 2024-09-17T23:53:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 9 2024-09-17T23:53:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd8 2024-09-17T23:53:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary9 2024-09-17T23:53:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary9 2024-09-17T23:53:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-17T23:53:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary9 2024-09-17T23:53:29.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-17T23:53:29.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:53:29.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-17T23:53:29.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:53:29.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-17T23:53:29.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:53:29.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-17T23:53:29.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:53:29.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-17T23:53:29.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:53:29.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:53:29.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-17T23:53:29.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-17T23:53:29.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-17T23:53:29.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:53:29.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:53:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:53:29.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-17T23:53:29.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-17T23:53:29.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:54:27.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=88b0c700ace24053b2b87edc2908b0cc 2024-09-17T23:54:27.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary5 2024-09-17T23:54:29.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=3f067af8cb9cea17873cca3d49a66476 2024-09-17T23:54:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=d0db9af3da06445b4c07a0ae49024aab 2024-09-17T23:54:29.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary6 2024-09-17T23:54:29.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary1 2024-09-17T23:54:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=5781acfad3ee324afd89c4689098b824 2024-09-17T23:54:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary8 2024-09-17T23:54:30.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=1962c7518d6ab6f913ce73c2ea2cf486 2024-09-17T23:54:30.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary7 2024-09-17T23:54:30.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=494d486c3a09596cee4366e7c5ee33e1 2024-09-17T23:54:30.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary2 2024-09-17T23:54:30.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=cc57279ca79cc809c1912c3a5e3f0fb5 2024-09-17T23:54:30.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary9 2024-09-17T23:54:31.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary1 2024-09-17T23:54:31.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:31.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:31.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:54:31.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary1 2024-09-17T23:54:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary8 2024-09-17T23:54:31.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:31.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:31.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:54:31.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary8 2024-09-17T23:54:31.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary6 2024-09-17T23:54:31.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:31.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:31.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:54:31.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary6 2024-09-17T23:54:31.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary5 2024-09-17T23:54:31.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:31.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:31.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:54:31.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary5 2024-09-17T23:54:31.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary7 2024-09-17T23:54:31.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:31.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:31.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:54:31.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary7 2024-09-17T23:54:31.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary2 2024-09-17T23:54:31.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:54:31.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary2 2024-09-17T23:54:32.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=ecef5b9d60c6001bc44657976b609dff 2024-09-17T23:54:32.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary10 2024-09-17T23:54:32.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=59100f74c7d3932e731a64127faa3e32 2024-09-17T23:54:32.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary4 2024-09-17T23:54:32.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary9 2024-09-17T23:54:32.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:32.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:32.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:54:32.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary9 2024-09-17T23:54:32.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary10 2024-09-17T23:54:32.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:32.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:32.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:54:32.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary10 2024-09-17T23:54:32.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary4 2024-09-17T23:54:32.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:32.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:32.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:54:32.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary4 2024-09-17T23:54:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-17T23:54:33.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-17T23:54:33.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-17T23:54:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-17T23:54:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+unknown 2024-09-17T23:54:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:54:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:33.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-17T23:54:33.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+unknown 2024-09-17T23:54:33.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:33.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:33.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:54:33.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:33.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:33.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:33.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:33.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:33.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+unknown 2024-09-17T23:54:33.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:33.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:33.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:54:33.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:33.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:33.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:33.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:33.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+unknown 2024-09-17T23:54:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:54:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:33.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:33.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+unknown 2024-09-17T23:54:33.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:33.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:33.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:54:33.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:33.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:33.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:33.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:33.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:33.268 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:54:33.259+0000 7fbbe8b19640 -1 reset not still connected to 0x561a087565b0 2024-09-17T23:54:33.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=90b2180afcb0e2c690057f6a2ff47934 2024-09-17T23:54:33.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary3 2024-09-17T23:54:34.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-17T23:54:34.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-17T23:54:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+unknown 2024-09-17T23:54:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:54:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:34.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-17T23:54:34.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+unknown 2024-09-17T23:54:34.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:54:34.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:34.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:34.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-17T23:54:34.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+unknown 2024-09-17T23:54:34.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:54:34.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:34.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+unknown 2024-09-17T23:54:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:54:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:34.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:34.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-17T23:54:34.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-17T23:54:34.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:54:34.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:34.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:54:34.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:34.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-17T23:54:34.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-17T23:54:34.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:54:34.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:34.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:34.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-17T23:54:34.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:34.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-17T23:54:34.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:34.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-17T23:54:34.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:34.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-17T23:54:34.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:34.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-17T23:54:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-17T23:54:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 2a0b583c-8984-4fe1-a8b4-539244bbc29d 2024-09-17T23:54:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:54:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:54:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2119680.0,"last_snapshot_bytes":2568192,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617045,"remote_snapshot_timestamp":1726617045,"replay_state":"idle"} 2024-09-17T23:54:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:09' 2024-09-17T23:54:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:34.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-17T23:54:34.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 551a4170-af79-4f62-9088-7d79bb565dcf 2024-09-17T23:54:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:54:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:54:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2140160.0,"last_snapshot_bytes":2527232,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617044,"remote_snapshot_timestamp":1726617044,"replay_state":"idle"} 2024-09-17T23:54:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:09' 2024-09-17T23:54:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:34.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-17T23:54:34.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 52d5d797-77fc-4c42-8bb6-0d80760c93da 2024-09-17T23:54:34.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:34.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:34.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:34.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:34.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:34.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:54:34.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:54:34.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3328000.0,"last_snapshot_bytes":4329472,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617040,"remote_snapshot_timestamp":1726617040,"replay_state":"idle"} 2024-09-17T23:54:34.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:09' 2024-09-17T23:54:34.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:34.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-17T23:54:34.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 6acb2ebc-f153-4559-98ce-7aeb7798bb9f 2024-09-17T23:54:34.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:34.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:34.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:34.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:34.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:34.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:54:34.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:54:34.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2131968.0,"last_snapshot_bytes":1687552,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617048,"remote_snapshot_timestamp":1726617048,"replay_state":"idle"} 2024-09-17T23:54:34.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:09' 2024-09-17T23:54:34.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:34.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-17T23:54:34.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a1cbc563-970b-461d-87b5-fa24b669fb62 2024-09-17T23:54:34.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:34.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:34.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:34.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:34.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:34.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:54:34.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:54:34.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":968704.0,"last_snapshot_bytes":1761280,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617046,"remote_snapshot_timestamp":1726617046,"replay_state":"idle"} 2024-09-17T23:54:34.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:09' 2024-09-17T23:54:34.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:34.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:34.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:34.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:34.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:34.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:34.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:34.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:34.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:34.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:34.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:34.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-17T23:54:34.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:34.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:34.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-17T23:54:34.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:34.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:34.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:34.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:34.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:34.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:34.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:34.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-17T23:54:34.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:34.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:34.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-17T23:54:34.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:34.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:34.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:34.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:34.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-17T23:54:34.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:34.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:34.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:34.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:34.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:34.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:34.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:34.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:34.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:34.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:34.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:34.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:34.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:34.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:34.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:34.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:34.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:34.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:34.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:34.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:54:34.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:54:34.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:34.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:54:34.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:34.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:34.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:54:34.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:34.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:34.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:34.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:34.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:34.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:34.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:34.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:34.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:54:34.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:34.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:34.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:34.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary3 2024-09-17T23:54:34.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:34.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:54:34.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary3 2024-09-17T23:54:34.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}2024-09-17 23:54:09image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34' 2024-09-17T23:54:34.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:34.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-17T23:54:34.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}2024-09-17 23:54:09image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34' 2024-09-17T23:54:34.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:34.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-17T23:54:34.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}2024-09-17 23:54:09image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34' 2024-09-17T23:54:34.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:34.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-17T23:54:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}2024-09-17 23:54:09image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34' 2024-09-17T23:54:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}2024-09-17 23:54:09image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34' 2024-09-17T23:54:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-17T23:54:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-17T23:54:34.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:34.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:34.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:34.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:34.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:34.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-17T23:54:34.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-17T23:54:34.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-17T23:54:34.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-17T23:54:34.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-17T23:54:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:34.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary8 up+unknown 2024-09-17T23:54:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:54:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:34.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:34.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:34.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:34.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary6 up+unknown 2024-09-17T23:54:34.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:34.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:54:34.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:34.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:34.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:34.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:34.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:34.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:34.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary1 up+unknown 2024-09-17T23:54:34.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:34.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:54:34.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:34.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary7 up+unknown 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary5 up+unknown 2024-09-17T23:54:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:54:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:35.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-17T23:54:35.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:35.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:54:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-17T23:54:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-17T23:54:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:54:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-17T23:54:35.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-17T23:54:35.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:54:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-17T23:54:35.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-17T23:54:35.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:35.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:54:35.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-17T23:54:35.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-17T23:54:35.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 7fdefe51-09c9-4400-ae06-6ebd475db1ce 2024-09-17T23:54:35.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:35.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:35.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:35.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:35.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:35.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:54:35.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:35.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:35.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:35.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:35.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:35.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:35.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-17T23:54:35.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:35.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:35.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:35.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:35.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:35.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:35.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:35.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:35.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:54:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:35.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-17T23:54:35.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8f9eb258-9a55-421c-878d-73e0cc38ad4b 2024-09-17T23:54:35.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:35.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:35.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:35.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:35.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:35.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:54:35.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:35.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:35.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:35.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:35.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:35.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:35.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-17T23:54:35.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:35.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:35.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:35.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:35.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:35.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:35.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:35.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:54:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-17T23:54:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 7cb937fc-9ee4-4d15-a2c2-54ef26b55f87 2024-09-17T23:54:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:35.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:54:35.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:35.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:35.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:35.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:35.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:35.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:35.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-17T23:54:35.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:35.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:35.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:35.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:35.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:35.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:35.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:35.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:35.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:35.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:35.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:35.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:54:35.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:35.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-17T23:54:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 969f453f-b516-4208-ac2e-cf5b99921fb5 2024-09-17T23:54:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:35.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:54:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-17T23:54:35.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:35.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:35.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:35.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:35.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:35.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:54:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:35.313 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-17T23:54:35.314+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d67ce42a90 2024-09-17T23:54:35.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}2024-09-17 23:54:09image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34' 2024-09-17T23:54:35.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:35.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-17T23:54:35.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:35.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-17T23:54:35.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:35.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:35.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:35.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:35.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:35.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:35.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary10 up+unknown 2024-09-17T23:54:35.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:35.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:54:35.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:35.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:35.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}2024-09-17 23:54:09image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34' 2024-09-17T23:54:35.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:35.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-17T23:54:35.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:35.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-17T23:54:35.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:35.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:35.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:35.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:35.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:35.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:35.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary2 up+unknown 2024-09-17T23:54:35.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:35.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:54:35.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:35.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:35.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}2024-09-17 23:54:09image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34' 2024-09-17T23:54:35.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:35.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-17T23:54:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}2024-09-17 23:54:09image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-17 23:54:34' 2024-09-17T23:54:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-17T23:54:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:35.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-17T23:54:35.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:35.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-17T23:54:35.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:35.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:35.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:35.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:35.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:35.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:35.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary9 up+unknown 2024-09-17T23:54:35.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:35.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:54:35.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:35.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:35.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:35.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:35.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:35.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:35.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:35.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:35.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary4 up+unknown 2024-09-17T23:54:35.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:35.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:54:35.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:35.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary8 up+unknown '' '' 2024-09-17T23:54:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary6 up+unknown '' '' 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary5 up+unknown '' '' 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary7 up+unknown '' '' 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary1 up+unknown '' '' 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:54:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary6 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary8 2024-09-17T23:54:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary1 2024-09-17T23:54:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary5 2024-09-17T23:54:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary7 2024-09-17T23:54:36.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-17T23:54:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+unknown 2024-09-17T23:54:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:54:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:36.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-17T23:54:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 2a0b583c-8984-4fe1-a8b4-539244bbc29d 2024-09-17T23:54:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-17T23:54:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:36.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:36.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:36.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-17T23:54:36.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:36.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:36.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:36.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-17T23:54:36.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a1cbc563-970b-461d-87b5-fa24b669fb62 2024-09-17T23:54:36.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:36.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:36.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:36.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:36.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-17T23:54:36.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 52d5d797-77fc-4c42-8bb6-0d80760c93da 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:36.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:36.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:36.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:36.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-17T23:54:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-17T23:54:36.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:36.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:36.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary6 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:36.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:54:36.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:36.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-17T23:54:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 551a4170-af79-4f62-9088-7d79bb565dcf 2024-09-17T23:54:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:36.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:36.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-17T23:54:36.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:36.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:36.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:36.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:36.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:36.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:36.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:36.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:36.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:36.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-17T23:54:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 6acb2ebc-f153-4559-98ce-7aeb7798bb9f 2024-09-17T23:54:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-17T23:54:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:36.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:36.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:36.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:36.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:36.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:36.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-17T23:54:36.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:36.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:36.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:36.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:36.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary7 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:36.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:54:36.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:36.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:36.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary1 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:54:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:36.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-17T23:54:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:36.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:36.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:36.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:36.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:36.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:36.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary5 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:36.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:36.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary8 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:54:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:36.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:36.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}42614264mirror.0smithi0492024-09-17 23:54:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-17 23:54:34image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34' 2024-09-17T23:54:36.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:36.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-17T23:54:36.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-17T23:54:36.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}42614264mirror.0smithi0492024-09-17 23:54:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-17 23:54:34image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34' 2024-09-17T23:54:36.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:36.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-17T23:54:36.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}42614264mirror.0smithi0492024-09-17 23:54:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-17 23:54:34image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34' 2024-09-17T23:54:36.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:36.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-17T23:54:36.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:36.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:36.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-17T23:54:36.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary8 2024-09-17T23:54:36.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:54:36.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-17T23:54:36.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary8 2024-09-17T23:54:36.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-17T23:54:36.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:36.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}42614264mirror.0smithi0492024-09-17 23:54:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-17 23:54:34image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34' 2024-09-17T23:54:36.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:36.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:36.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-17T23:54:36.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}42614264mirror.0smithi0492024-09-17 23:54:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-17 23:54:34image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34' 2024-09-17T23:54:36.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:36.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-17T23:54:36.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary1 2024-09-17T23:54:36.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:54:36.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-17T23:54:36.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary1 2024-09-17T23:54:36.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:36.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:36.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-17T23:54:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary6 2024-09-17T23:54:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:54:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-17T23:54:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary6 2024-09-17T23:54:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-17T23:54:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:36.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary7 2024-09-17T23:54:36.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:54:36.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-17T23:54:36.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary7 2024-09-17T23:54:36.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:36.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary5 2024-09-17T23:54:36.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:54:36.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-17T23:54:36.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary5 2024-09-17T23:54:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary10 up+unknown '' '' 2024-09-17T23:54:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:54:36.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:36.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:36.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary10 2024-09-17T23:54:36.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary2 up+unknown '' '' 2024-09-17T23:54:36.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:54:36.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:36.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:36.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary2 2024-09-17T23:54:36.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary9 up+unknown '' '' 2024-09-17T23:54:36.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:54:36.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:36.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:36.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary9 2024-09-17T23:54:36.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary4 up+unknown '' '' 2024-09-17T23:54:36.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:54:36.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:36.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:36.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary4 2024-09-17T23:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-17T23:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 7fdefe51-09c9-4400-ae06-6ebd475db1ce 2024-09-17T23:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-17T23:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:36.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:36.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:36.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:36.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:36.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-17T23:54:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:36.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-17T23:54:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 7cb937fc-9ee4-4d15-a2c2-54ef26b55f87 2024-09-17T23:54:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-17T23:54:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:36.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:36.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary10 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:36.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:54:36.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:36.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:36.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-17T23:54:36.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:36.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:36.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-17T23:54:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 969f453f-b516-4208-ac2e-cf5b99921fb5 2024-09-17T23:54:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-17T23:54:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:36.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:36.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:36.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-17T23:54:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary9 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:54:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-17T23:54:36.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8f9eb258-9a55-421c-878d-73e0cc38ad4b 2024-09-17T23:54:36.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:36.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:36.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-17T23:54:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34' 2024-09-17T23:54:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:36.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-17T23:54:36.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:36.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:36.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary4 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:36.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:54:36.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:36.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:36.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:36.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:36.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:36.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:36.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary2 up+unknown '2024-09-17 23:54:34' 2024-09-17T23:54:36.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:54:36.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:36.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:34' 2024-09-17T23:54:36.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:36.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:36.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:36.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-17T23:54:36.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-17T23:54:36.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-17T23:54:36.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-17T23:54:36.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+stopping_replayremote image demoted42614264mirror.0smithi0492024-09-17 23:54:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-17 23:54:34image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34' 2024-09-17T23:54:36.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:36.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-17T23:54:36.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:54:36.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-17T23:54:36.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}42614264mirror.0smithi0492024-09-17 23:54:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-17 23:54:34image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34' 2024-09-17T23:54:36.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:36.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-17T23:54:36.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-17T23:54:36.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617042,"remote_snapshot_timestamp":1726617042,"replay_state":"idle"}42614264mirror.0smithi0492024-09-17 23:54:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-17 23:54:34image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34' 2024-09-17T23:54:36.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:36.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-17T23:54:36.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:54:36.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-17T23:54:36.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-17T23:54:36.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-17T23:54:36.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:36.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:54:36.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:36.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-17T23:54:36.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-17T23:54:36.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary10 2024-09-17T23:54:36.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:54:36.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-17T23:54:36.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary10 2024-09-17T23:54:36.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:54:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-17 23:54:34image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+unknownremote image demoted42614264mirror.0smithi0492024-09-17 23:54:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:34' 2024-09-17T23:54:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:54:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-17T23:54:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-17T23:54:36.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:34' 2024-09-17T23:54:36.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-17T23:54:36.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-17T23:54:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary9 2024-09-17T23:54:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:54:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-17T23:54:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary9 2024-09-17T23:54:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary4 2024-09-17T23:54:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:54:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-17T23:54:36.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary4 2024-09-17T23:54:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary2 2024-09-17T23:54:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:54:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-17T23:54:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary2 2024-09-17T23:54:36.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd0 2024-09-17T23:54:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-17T23:54:36.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:54:36.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd1 2024-09-17T23:54:36.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-17T23:54:36.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:54:36.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd2 2024-09-17T23:54:36.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-17T23:54:36.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:54:36.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd3 2024-09-17T23:54:36.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-17T23:54:36.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:54:37.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd4 2024-09-17T23:54:37.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:54:37.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-17T23:54:37.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-17T23:54:37.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:37.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:54:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:37.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-17T23:54:37.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-17T23:54:37.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 3643a8f6-f271-4297-95a3-71f0728fb1fa 2024-09-17T23:54:37.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-17T23:54:37.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-17T23:54:37.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:37.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:34 2024-09-17T23:54:37.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:37.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:54:37.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:37.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-17T23:54:37.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:36' 2024-09-17T23:54:37.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:37.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-17T23:54:37.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:37.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-17T23:54:37.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-17T23:54:37.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-17T23:54:37.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:54:37.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-17T23:54:37.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-17T23:54:37.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-17T23:54:37.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:54:37.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-17T23:54:37.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:54:37.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-17T23:54:37.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:54:37.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-17T23:54:37.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd5 2024-09-17T23:54:37.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd6 2024-09-17T23:54:37.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-17T23:54:37.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:54:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd7 2024-09-17T23:54:37.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-17T23:54:37.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:54:37.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-17T23:54:37.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:54:37.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd8 2024-09-17T23:54:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:54:37.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-17T23:54:39.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-17T23:54:39.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:54:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-17T23:54:39.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-17T23:54:39.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 3643a8f6-f271-4297-95a3-71f0728fb1fa 2024-09-17T23:54:39.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:39.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:39.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:39.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:39 2024-09-17T23:54:39.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:39.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:54:39.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:39.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:39.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:39' 2024-09-17T23:54:39.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:39.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:39.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:39.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-17T23:54:39.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:39.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:39.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:39.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:39.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:39.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:39.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:39.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:39.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:39' 2024-09-17T23:54:39.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+unknown '2024-09-17 23:54:39' 2024-09-17T23:54:39.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:54:39.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:39.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:54:39.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:39.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:39' 2024-09-17T23:54:39.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:39.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:39.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:39.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:54:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:54:39image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:54:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:54:39image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:54:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:54:39image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+unknownremote image is not primary42544257mirror.0smithi0492024-09-17 23:54:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-17 23:54:39image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:54:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:54:39image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:54:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:54:39image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:54:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:54:39image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:54:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:54:39image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:54:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:54:39image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:54:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:54:39' 2024-09-17T23:54:39.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:39.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-17T23:54:39.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:39' 2024-09-17T23:54:39.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-17T23:54:39.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:39.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:39.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:39.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:39.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary3 up+unknown 2024-09-17T23:54:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:54:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:54:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:54:40.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary3 up+unknown '' '' 2024-09-17T23:54:40.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:40.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:40.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:54:40.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:40.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:54:40.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:54:40.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:40.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:40.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary3 2024-09-17T23:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-17T23:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 3643a8f6-f271-4297-95a3-71f0728fb1fa 2024-09-17T23:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:39 2024-09-17T23:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-17T23:54:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-17T23:54:40.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-17T23:54:40.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:54:39' 2024-09-17T23:54:40.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-17T23:54:40.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-17T23:54:40.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:54:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-17T23:54:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:54:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:54:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:41.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:54:41.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:54:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:54:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:54:41.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:39' 2024-09-17T23:54:41.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary3 up+unknown '2024-09-17 23:54:39' 2024-09-17T23:54:41.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:41.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:41.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:54:41.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-17T23:54:41.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:54:39' 2024-09-17T23:54:41.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:54:41.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:54:41.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-17T23:54:41.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1942614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-17 23:54:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}2024-09-17 23:54:39image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-17 23:54:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}2024-09-17 23:54:39image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-17 23:54:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}2024-09-17 23:54:39image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+unknownremote image is not primary42614264mirror.0smithi0492024-09-17 23:54:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-17 23:54:39image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-17 23:54:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}2024-09-17 23:54:39image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-17 23:54:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}2024-09-17 23:54:39image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-17 23:54:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}2024-09-17 23:54:39image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-17 23:54:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}2024-09-17 23:54:39image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-17 23:54:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}2024-09-17 23:54:39image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-17 23:54:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}2024-09-17 23:54:39' 2024-09-17T23:54:41.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:54:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-17T23:54:41.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:54:39' 2024-09-17T23:54:41.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-17T23:54:41.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-17T23:54:41.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-17T23:54:41.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-17T23:54:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-17T23:54:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:54:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary3 2024-09-17T23:54:41.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:54:41.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:54:41.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:54:41.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-17T23:54:41.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary3 2024-09-17T23:54:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-17T23:54:42.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:54:42.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-17T23:54:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd9 2024-09-17T23:54:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-17T23:54:43.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-17T23:56:25.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=5781acfad3ee324afd89c4689098b824 2024-09-17T23:56:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd1 2024-09-17T23:56:25.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=d0db9af3da06445b4c07a0ae49024aab 2024-09-17T23:56:25.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd4 2024-09-17T23:56:26.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 5781acfad3ee324afd89c4689098b824 != \5\7\8\1\a\c\f\a\d\3\e\e\3\2\4\a\f\d\8\9\c\4\6\8\9\0\9\8\b\8\2\4 ]] 2024-09-17T23:56:26.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=1962c7518d6ab6f913ce73c2ea2cf486 2024-09-17T23:56:26.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd3 2024-09-17T23:56:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=59100f74c7d3932e731a64127faa3e32 2024-09-17T23:56:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd5 2024-09-17T23:56:27.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ d0db9af3da06445b4c07a0ae49024aab != \d\0\d\b\9\a\f\3\d\a\0\6\4\4\5\b\4\c\0\7\a\0\a\e\4\9\0\2\4\a\a\b ]] 2024-09-17T23:56:27.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=3f067af8cb9cea17873cca3d49a66476 2024-09-17T23:56:27.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd0 2024-09-17T23:56:27.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=88b0c700ace24053b2b87edc2908b0cc 2024-09-17T23:56:27.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd2 2024-09-17T23:56:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 1962c7518d6ab6f913ce73c2ea2cf486 != \1\9\6\2\c\7\5\1\8\d\6\a\b\6\f\9\1\3\c\e\7\3\c\2\e\a\2\c\f\4\8\6 ]] 2024-09-17T23:56:27.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 59100f74c7d3932e731a64127faa3e32 != \5\9\1\0\0\f\7\4\c\7\d\3\9\3\2\e\7\3\1\a\6\4\1\2\7\f\a\a\3\e\3\2 ]] 2024-09-17T23:56:28.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=cc57279ca79cc809c1912c3a5e3f0fb5 2024-09-17T23:56:28.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd7 2024-09-17T23:56:28.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 3f067af8cb9cea17873cca3d49a66476 != \3\f\0\6\7\a\f\8\c\b\9\c\e\a\1\7\8\7\3\c\c\a\3\d\4\9\a\6\6\4\7\6 ]] 2024-09-17T23:56:28.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-17T23:56:28.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 37728 2024-09-17T23:56:28.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=ecef5b9d60c6001bc44657976b609dff 2024-09-17T23:56:28.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd8 2024-09-17T23:56:28.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=494d486c3a09596cee4366e7c5ee33e1 2024-09-17T23:56:28.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd6 2024-09-17T23:56:28.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 88b0c700ace24053b2b87edc2908b0cc != \8\8\b\0\c\7\0\0\a\c\e\2\4\0\5\3\b\2\b\8\7\e\d\c\2\9\0\8\b\0\c\c ]] 2024-09-17T23:56:28.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=90b2180afcb0e2c690057f6a2ff47934 2024-09-17T23:56:28.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd9 2024-09-17T23:56:28.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ cc57279ca79cc809c1912c3a5e3f0fb5 != \c\c\5\7\2\7\9\c\a\7\9\c\c\8\0\9\c\1\9\1\2\c\3\a\5\e\3\f\0\f\b\5 ]] 2024-09-17T23:56:28.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ ecef5b9d60c6001bc44657976b609dff != \e\c\e\f\5\b\9\d\6\0\c\6\0\0\1\b\c\4\4\6\5\7\9\7\6\b\6\0\9\d\f\f ]] 2024-09-17T23:56:29.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 494d486c3a09596cee4366e7c5ee33e1 != \4\9\4\d\4\8\6\c\3\a\0\9\5\9\6\c\e\e\4\3\6\6\e\7\c\5\e\e\3\3\e\1 ]] 2024-09-17T23:56:29.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-17T23:56:29.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 37729 2024-09-17T23:56:29.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 90b2180afcb0e2c690057f6a2ff47934 != \9\0\b\2\1\8\0\a\f\c\b\0\e\2\c\6\9\0\0\5\7\f\6\a\2\f\f\4\7\9\3\4 ]] 2024-09-17T23:56:29.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-17T23:56:29.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 37732 2024-09-17T23:56:29.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-17T23:56:29.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 37734 2024-09-17T23:56:29.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-17T23:56:29.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 37736 2024-09-17T23:56:29.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-17T23:56:29.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 37738 2024-09-17T23:56:29.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-17T23:56:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 37740 2024-09-17T23:56:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-17T23:56:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 37742 2024-09-17T23:56:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-17T23:56:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 37743 2024-09-17T23:56:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-17T23:56:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:56:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-17T23:56:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary1 2024-09-17T23:56:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:56:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-17T23:56:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+replaying 2024-09-17T23:56:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:56:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:56:30.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+replaying '' '' 2024-09-17T23:56:30.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:30.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:30.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:56:30.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:30.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:30.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:30.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-17T23:56:30.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-17T23:56:30.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 52d5d797-77fc-4c42-8bb6-0d80760c93da 2024-09-17T23:56:30.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:56:30.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":150683648.0,"last_snapshot_bytes":150683648,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"} 2024-09-17T23:56:30.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:56:30.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:09 2024-09-17T23:56:30.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:56:30.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:56:30.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-17T23:56:30.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-17T23:56:30.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:29' 2024-09-17T23:56:30.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-17T23:56:30.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-17T23:56:30.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:56:30.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":150683648.0,"last_snapshot_bytes":150683648,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"} 2024-09-17T23:56:30.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:56:30.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:30.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:56:30.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:30.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:56:30.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:56:30.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:56:30.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:56:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:09' 2024-09-17T23:56:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+replaying '2024-09-17 23:56:09' 2024-09-17T23:56:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:56:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:56:09' 2024-09-17T23:56:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:56:30.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary152d5d797-77fc-4c42-8bb6-0d80760c93daup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":150683648.0,"last_snapshot_bytes":150683648,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:29image-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:29image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:29image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72355840.0,"last_snapshot_bytes":72355840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617282,"remote_snapshot_timestamp":1726617282,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:29image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70291456.0,"last_snapshot_bytes":70291456,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:29image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:29image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70283264.0,"last_snapshot_bytes":70283264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:29image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:29image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70336512.0,"last_snapshot_bytes":70336512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:29image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":107872256.0,"last_snapshot_bytes":107872256,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:29' 2024-09-17T23:56:30.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:56:30.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-17T23:56:31.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:09' 2024-09-17T23:56:31.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-17T23:56:31.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-17T23:56:31.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:31.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-17T23:56:31.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:31.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:31.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:31.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary1 2024-09-17T23:56:31.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:56:31.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:31.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:56:31.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary1 2024-09-17T23:56:31.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary1 2024-09-17T23:56:32.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-17T23:56:32.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary1 2024-09-17T23:56:32.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:32.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:32.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:56:32.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:32.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-17T23:56:32.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary1 2024-09-17T23:56:32.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:32.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary1 2024-09-17T23:56:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:56:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-17T23:56:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary2 2024-09-17T23:56:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:56:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-17T23:56:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+replaying 2024-09-17T23:56:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:56:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:56:34.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+replaying '' '' 2024-09-17T23:56:34.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:34.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:34.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:56:34.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:34.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:34.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:34.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:34.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:34.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-17T23:56:34.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-17T23:56:34.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8f9eb258-9a55-421c-878d-73e0cc38ad4b 2024-09-17T23:56:34.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:56:34.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"} 2024-09-17T23:56:34.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:56:34.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:09 2024-09-17T23:56:34.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:56:34.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:56:34.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-17T23:56:34.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-17T23:56:34.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:34' 2024-09-17T23:56:34.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-17T23:56:34.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-17T23:56:34.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:56:34.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"} 2024-09-17T23:56:34.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:56:34.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:34.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:56:34.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:34.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:56:34.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:56:34.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:56:34.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:56:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:09' 2024-09-17T23:56:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+replaying '2024-09-17 23:56:09' 2024-09-17T23:56:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:56:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:56:09' 2024-09-17T23:56:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:56:34.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary28f9eb258-9a55-421c-878d-73e0cc38ad4bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72355840.0,"last_snapshot_bytes":72355840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617282,"remote_snapshot_timestamp":1726617282,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70291456.0,"last_snapshot_bytes":70291456,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70283264.0,"last_snapshot_bytes":70283264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70336512.0,"last_snapshot_bytes":70336512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":107872256.0,"last_snapshot_bytes":107872256,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34' 2024-09-17T23:56:34.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:56:34.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-17T23:56:34.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:09' 2024-09-17T23:56:34.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-17T23:56:34.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-17T23:56:34.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:34.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-17T23:56:34.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:34.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:34.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:34.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary2 2024-09-17T23:56:34.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:56:34.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:34.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:56:34.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary2 2024-09-17T23:56:34.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary2 2024-09-17T23:56:35.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-17T23:56:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary2 2024-09-17T23:56:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:56:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-17T23:56:35.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary2 2024-09-17T23:56:35.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:35.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary2 2024-09-17T23:56:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:56:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-17T23:56:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary3 2024-09-17T23:56:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:56:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-17T23:56:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+replaying 2024-09-17T23:56:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:35.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:56:35.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:35.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:35.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:35.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:35.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:56:36.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+replaying '' '' 2024-09-17T23:56:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:56:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-17T23:56:37.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-17T23:56:37.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 3643a8f6-f271-4297-95a3-71f0728fb1fa 2024-09-17T23:56:37.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:56:37.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72355840.0,"last_snapshot_bytes":72355840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617282,"remote_snapshot_timestamp":1726617282,"replay_state":"idle"} 2024-09-17T23:56:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:56:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:09 2024-09-17T23:56:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:56:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:56:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-17T23:56:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-17T23:56:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:34' 2024-09-17T23:56:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-17T23:56:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-17T23:56:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:56:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72355840.0,"last_snapshot_bytes":72355840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617282,"remote_snapshot_timestamp":1726617282,"replay_state":"idle"} 2024-09-17T23:56:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:56:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:56:37.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:56:37.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:56:37.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:56:37.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:56:37.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:09' 2024-09-17T23:56:37.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+replaying '2024-09-17 23:56:09' 2024-09-17T23:56:37.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:37.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:37.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:56:37.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:37.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:56:09' 2024-09-17T23:56:37.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:37.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:37.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:56:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK842544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary33643a8f6-f271-4297-95a3-71f0728fb1faup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72355840.0,"last_snapshot_bytes":72355840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617282,"remote_snapshot_timestamp":1726617282,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70291456.0,"last_snapshot_bytes":70291456,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70283264.0,"last_snapshot_bytes":70283264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70336512.0,"last_snapshot_bytes":70336512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":107872256.0,"last_snapshot_bytes":107872256,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:34' 2024-09-17T23:56:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:56:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-17T23:56:37.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:09' 2024-09-17T23:56:37.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-17T23:56:37.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-17T23:56:37.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:37.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-17T23:56:37.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:37.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:37.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:37.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary3 2024-09-17T23:56:37.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:56:37.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:37.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:56:37.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary3 2024-09-17T23:56:37.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary3 2024-09-17T23:56:38.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-17T23:56:38.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary3 2024-09-17T23:56:38.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:38.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:38.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:56:38.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:38.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-17T23:56:38.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary3 2024-09-17T23:56:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:38.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary3 2024-09-17T23:56:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:56:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-17T23:56:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary4 2024-09-17T23:56:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:56:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-17T23:56:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+replaying 2024-09-17T23:56:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:56:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:39.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:39.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:56:40.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+replaying '' '' 2024-09-17T23:56:40.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:40.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:56:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-17T23:56:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-17T23:56:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 969f453f-b516-4208-ac2e-cf5b99921fb5 2024-09-17T23:56:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:56:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70291456.0,"last_snapshot_bytes":70291456,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"} 2024-09-17T23:56:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:56:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:39 2024-09-17T23:56:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:56:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:56:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-17T23:56:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-17T23:56:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:39' 2024-09-17T23:56:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-17T23:56:40.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-17T23:56:40.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:56:40.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70291456.0,"last_snapshot_bytes":70291456,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"} 2024-09-17T23:56:40.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:56:40.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:40.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:56:40.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:40.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:56:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:56:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:56:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:56:40.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:40.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+replaying '2024-09-17 23:56:39' 2024-09-17T23:56:40.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:40.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:56:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:56:39' 2024-09-17T23:56:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:56:40.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39image-primary4969f453f-b516-4208-ac2e-cf5b99921fb5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70291456.0,"last_snapshot_bytes":70291456,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70283264.0,"last_snapshot_bytes":70283264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70336512.0,"last_snapshot_bytes":70336512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":107872256.0,"last_snapshot_bytes":107872256,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39' 2024-09-17T23:56:40.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:56:40.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-17T23:56:40.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:40.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-17T23:56:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-17T23:56:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:40.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-17T23:56:40.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:40.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:40.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:40.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary4 2024-09-17T23:56:40.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:56:40.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:40.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:56:40.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary4 2024-09-17T23:56:40.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary4 2024-09-17T23:56:41.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-17T23:56:41.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary4 2024-09-17T23:56:41.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:41.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:41.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:56:41.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:41.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-17T23:56:41.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary4 2024-09-17T23:56:42.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:42.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary4 2024-09-17T23:56:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:56:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-17T23:56:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary5 2024-09-17T23:56:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:56:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-17T23:56:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+replaying 2024-09-17T23:56:42.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:42.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:42.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:56:42.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:42.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:42.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:42.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:42.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:56:43.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+replaying '' '' 2024-09-17T23:56:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:56:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-17T23:56:43.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-17T23:56:43.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 551a4170-af79-4f62-9088-7d79bb565dcf 2024-09-17T23:56:43.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:56:43.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"} 2024-09-17T23:56:43.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:56:43.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:39 2024-09-17T23:56:43.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:56:43.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:56:43.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-17T23:56:43.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-17T23:56:43.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:39' 2024-09-17T23:56:43.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-17T23:56:43.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-17T23:56:43.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:56:43.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"} 2024-09-17T23:56:43.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:56:43.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:43.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:56:43.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:43.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:56:43.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:56:43.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:56:43.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:56:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+replaying '2024-09-17 23:56:39' 2024-09-17T23:56:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:56:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:56:39' 2024-09-17T23:56:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:56:43.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK642544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39image-primary5551a4170-af79-4f62-9088-7d79bb565dcfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70283264.0,"last_snapshot_bytes":70283264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70336512.0,"last_snapshot_bytes":70336512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":107872256.0,"last_snapshot_bytes":107872256,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:39' 2024-09-17T23:56:43.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:56:43.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-17T23:56:43.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:43.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-17T23:56:43.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-17T23:56:43.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:43.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-17T23:56:43.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:43.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:43.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:43.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary5 2024-09-17T23:56:43.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:56:43.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:43.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:56:43.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary5 2024-09-17T23:56:43.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary5 2024-09-17T23:56:44.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-17T23:56:44.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary5 2024-09-17T23:56:44.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:44.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:44.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:56:44.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:44.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-17T23:56:44.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary5 2024-09-17T23:56:44.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:44.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary5 2024-09-17T23:56:44.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:56:44.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-17T23:56:44.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary6 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+replaying 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:56:45.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+replaying '' '' 2024-09-17T23:56:45.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:45.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:45.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:56:45.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:45.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:45.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:45.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:45.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:45.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-17T23:56:45.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-17T23:56:45.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 2a0b583c-8984-4fe1-a8b4-539244bbc29d 2024-09-17T23:56:45.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:56:45.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70283264.0,"last_snapshot_bytes":70283264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"} 2024-09-17T23:56:45.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:56:45.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:39 2024-09-17T23:56:45.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:56:45.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:56:45.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-17T23:56:45.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-17T23:56:45.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:44' 2024-09-17T23:56:45.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-17T23:56:45.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-17T23:56:45.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:56:45.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70283264.0,"last_snapshot_bytes":70283264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"} 2024-09-17T23:56:45.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:56:45.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:45.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:56:45.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:45.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:56:45.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:56:45.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:56:45.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:56:45.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:45.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+replaying '2024-09-17 23:56:39' 2024-09-17T23:56:45.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:45.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:45.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:56:45.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:45.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:56:39' 2024-09-17T23:56:45.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:45.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:45.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:56:46.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:44image-primary62a0b583c-8984-4fe1-a8b4-539244bbc29dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70283264.0,"last_snapshot_bytes":70283264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:44image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:44image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70336512.0,"last_snapshot_bytes":70336512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:44image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":107872256.0,"last_snapshot_bytes":107872256,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:44' 2024-09-17T23:56:46.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:56:46.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-17T23:56:46.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:46.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-17T23:56:46.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-17T23:56:46.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:46.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-17T23:56:46.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:46.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:46.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:46.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary6 2024-09-17T23:56:46.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:56:46.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:46.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:56:46.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary6 2024-09-17T23:56:46.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary6 2024-09-17T23:56:47.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-17T23:56:47.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary6 2024-09-17T23:56:47.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:47.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:47.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:56:47.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:47.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-17T23:56:47.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary6 2024-09-17T23:56:48.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:48.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary6 2024-09-17T23:56:48.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:56:48.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-17T23:56:48.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary7 2024-09-17T23:56:48.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:48.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:48.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:56:48.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-17T23:56:48.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+replaying 2024-09-17T23:56:48.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:56:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:56:49.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+replaying '' '' 2024-09-17T23:56:49.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:49.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:49.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:56:49.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:49.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:49.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:49.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:49.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:49.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-17T23:56:49.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-17T23:56:49.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a1cbc563-970b-461d-87b5-fa24b669fb62 2024-09-17T23:56:49.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:56:49.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"} 2024-09-17T23:56:49.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:56:49.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:39 2024-09-17T23:56:49.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:56:49.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:56:49.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-17T23:56:49.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-17T23:56:49.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:44' 2024-09-17T23:56:49.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-17T23:56:49.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-17T23:56:49.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:56:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"} 2024-09-17T23:56:49.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:56:49.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:49.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:56:49.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:49.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:56:49.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:56:49.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:56:49.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:56:49.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+replaying '2024-09-17 23:56:39' 2024-09-17T23:56:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:56:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:56:39' 2024-09-17T23:56:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:49.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:56:49.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:49image-primary7a1cbc563-970b-461d-87b5-fa24b669fb62up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:49image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70336512.0,"last_snapshot_bytes":70336512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:49image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":107872256.0,"last_snapshot_bytes":107872256,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:49' 2024-09-17T23:56:49.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:56:49.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-17T23:56:49.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:49.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-17T23:56:49.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-17T23:56:49.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:49.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-17T23:56:49.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:49.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:49.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:49.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary7 2024-09-17T23:56:49.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:56:49.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:49.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:56:49.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary7 2024-09-17T23:56:49.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary7 2024-09-17T23:56:50.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-17T23:56:50.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary7 2024-09-17T23:56:50.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:50.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:50.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:56:50.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:50.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-17T23:56:50.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary7 2024-09-17T23:56:50.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:50.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary7 2024-09-17T23:56:50.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:56:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-17T23:56:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary8 2024-09-17T23:56:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:56:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-17T23:56:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+replaying 2024-09-17T23:56:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:56:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:56:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+replaying '' '' 2024-09-17T23:56:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:56:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-17T23:56:51.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-17T23:56:51.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 6acb2ebc-f153-4559-98ce-7aeb7798bb9f 2024-09-17T23:56:51.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:56:51.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70336512.0,"last_snapshot_bytes":70336512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"} 2024-09-17T23:56:51.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:56:51.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:39 2024-09-17T23:56:51.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:56:51.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:56:51.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-17T23:56:51.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-17T23:56:51.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:49' 2024-09-17T23:56:51.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-17T23:56:51.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-17T23:56:51.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:56:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70336512.0,"last_snapshot_bytes":70336512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"} 2024-09-17T23:56:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:56:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:56:51.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:51.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:56:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:56:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:56:51.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:56:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+replaying '2024-09-17 23:56:39' 2024-09-17T23:56:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:56:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:56:39' 2024-09-17T23:56:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:56:52.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK342544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:49image-primary86acb2ebc-f153-4559-98ce-7aeb7798bb9fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70336512.0,"last_snapshot_bytes":70336512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617276,"remote_snapshot_timestamp":1726617276,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:49image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":107872256.0,"last_snapshot_bytes":107872256,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:49' 2024-09-17T23:56:52.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:56:52.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-17T23:56:52.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:52.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-17T23:56:52.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-17T23:56:52.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:52.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-17T23:56:52.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary8 2024-09-17T23:56:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:56:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:56:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary8 2024-09-17T23:56:52.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary8 2024-09-17T23:56:54.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-17T23:56:54.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary8 2024-09-17T23:56:54.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:54.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:54.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:56:54.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:54.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-17T23:56:54.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary8 2024-09-17T23:56:54.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:54.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary8 2024-09-17T23:56:54.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:56:54.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-17T23:56:54.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary9 2024-09-17T23:56:54.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:54.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:54.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:56:54.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-17T23:56:54.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+replaying 2024-09-17T23:56:54.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:54.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:54.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:56:54.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:54.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:54.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:54.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:54.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:56:54.175 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-17T23:56:54.172+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-17T23:56:54.175 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-17T23:56:54.172+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-17T23:56:55.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+replaying '' '' 2024-09-17T23:56:55.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:55.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:55.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:56:55.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:55.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:55.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:55.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:55.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:55.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-17T23:56:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-17T23:56:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 7cb937fc-9ee4-4d15-a2c2-54ef26b55f87 2024-09-17T23:56:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:56:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":107872256.0,"last_snapshot_bytes":107872256,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"} 2024-09-17T23:56:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:56:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:39 2024-09-17T23:56:55.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:56:55.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:56:55.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-17T23:56:55.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-17T23:56:55.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:54' 2024-09-17T23:56:55.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-17T23:56:55.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-17T23:56:55.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:56:55.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":107872256.0,"last_snapshot_bytes":107872256,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"} 2024-09-17T23:56:55.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:56:55.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:55.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:56:55.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:55.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:56:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:56:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:56:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:56:55.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:55.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+replaying '2024-09-17 23:56:39' 2024-09-17T23:56:55.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:55.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:55.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:56:55.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:55.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:56:39' 2024-09-17T23:56:55.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:55.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:55.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:56:55.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:54image-primary97cb937fc-9ee4-4d15-a2c2-54ef26b55f87up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":107872256.0,"last_snapshot_bytes":107872256,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:54' 2024-09-17T23:56:55.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:56:55.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-17T23:56:55.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:55.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-17T23:56:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-17T23:56:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-17T23:56:55.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:55.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:55.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:55.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary9 2024-09-17T23:56:55.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:56:55.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:55.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:56:55.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary9 2024-09-17T23:56:55.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary9 2024-09-17T23:56:57.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-17T23:56:57.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary9 2024-09-17T23:56:57.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:57.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:57.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:56:57.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:57.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-17T23:56:57.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary9 2024-09-17T23:56:57.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:57.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary9 2024-09-17T23:56:57.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:56:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-17T23:56:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary10 2024-09-17T23:56:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:56:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-17T23:56:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+replaying 2024-09-17T23:56:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:56:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:57.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:57.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:57.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-17T23:56:58.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+replaying '' '' 2024-09-17T23:56:58.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:58.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:58.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:56:58.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:58.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-17T23:56:58.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-17T23:56:58.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:58.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:58.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-17T23:56:58.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-17T23:56:58.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 7fdefe51-09c9-4400-ae06-6ebd475db1ce 2024-09-17T23:56:58.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-17T23:56:58.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"} 2024-09-17T23:56:58.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-17T23:56:58.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:39 2024-09-17T23:56:58.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-17T23:56:58.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-17T23:56:58.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-17T23:56:58.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-17T23:56:58.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-17 23:56:54' 2024-09-17T23:56:58.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-17T23:56:58.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-17T23:56:58.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-17T23:56:58.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"} 2024-09-17T23:56:58.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-17T23:56:58.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:58.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-17T23:56:58.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:58.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-17T23:56:58.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-17T23:56:58.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-17T23:56:58.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-17T23:56:58.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:58.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+replaying '2024-09-17 23:56:39' 2024-09-17T23:56:58.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:58.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:58.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:56:58.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-17T23:56:58.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-17 23:56:39' 2024-09-17T23:56:58.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-17T23:56:58.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-17T23:56:58.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-17T23:56:58.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary107fdefe51-09c9-4400-ae06-6ebd475db1ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617277,"remote_snapshot_timestamp":1726617277,"replay_state":"idle"}42544257mirror.0smithi0492024-09-17 23:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-17 23:56:54' 2024-09-17T23:56:58.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-17T23:56:58.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-17T23:56:58.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-17 23:56:39' 2024-09-17T23:56:58.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-17T23:56:58.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-17T23:56:58.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-17T23:56:58.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-17T23:56:58.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-17T23:56:58.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:58.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:58.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary10 2024-09-17T23:56:58.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-17T23:56:58.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:58.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:56:58.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary10 2024-09-17T23:56:58.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary10 2024-09-17T23:56:59.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-17T23:56:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary10 2024-09-17T23:56:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:56:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-17T23:56:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-17T23:56:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary10 2024-09-17T23:56:59.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:59.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary10 2024-09-17T23:56:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..10} 2024-09-17T23:56:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS=() 2024-09-17T23:56:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS=() 2024-09-17T23:56:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS=() 2024-09-17T23:56:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS=() 2024-09-17T23:56:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RET=0 2024-09-17T23:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-17T23:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary1 2024-09-17T23:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary1 snapshot 10G 2024-09-17T23:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:56:59.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary1 10G 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:56:59.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary1 2024-09-17T23:56:59.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:56:59.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:59.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:56:59.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:56:59.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary1 snapshot 2024-09-17T23:56:59.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:56:59.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:56:59.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:56:59.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:56:59.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary1 snapshot 2024-09-17T23:57:01.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:57:01.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary1 2024-09-17T23:57:01.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary1': 2024-09-17T23:57:01.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:57:01.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:57:01.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:57:01.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1531a3da8e2b 2024-09-17T23:57:01.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1531a3da8e2b 2024-09-17T23:57:01.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:57:01.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:57:01.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:57:01.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:57:01.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:56:59 2024 2024-09-17T23:57:01.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:56:59 2024 2024-09-17T23:57:01.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:56:59 2024 2024-09-17T23:57:01.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:57:01.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:57:01.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 86e615f2-9a52-4cc1-a03c-46ae2effd0d8 2024-09-17T23:57:01.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:57:01.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:57:01.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-17T23:57:01.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd0 2024-09-17T23:57:01.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:57:01.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd0 2024-09-17T23:57:01.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:57:01.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:57:01.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:57:01.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: ca5bea4f-9e53-4403-b43a-a250b866041b 2024-09-17T23:57:01.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:57:01.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:57:01.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:01.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:57:01.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:57:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:57:01.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:57:01.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:01.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary1 2024-09-17T23:57:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd0 test-primary1 2024-09-17T23:57:01.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:57:01.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary1 2024-09-17T23:57:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:57:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary1 2024-09-17T23:57:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:57:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:57:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-17T23:57:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary2 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary2 snapshot 10G 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary1 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary2 10G 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary1 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary1 2024-09-17T23:57:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:57:01.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:01.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary2 2024-09-17T23:57:01.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:57:01.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:01.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:57:01.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:57:01.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary2 snapshot 2024-09-17T23:57:01.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:01.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:01.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:01.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:01.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary2 snapshot 2024-09-17T23:57:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 334 2024-09-17T23:57:02.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:02.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:57:02.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary1/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary1' 2024-09-17T23:57:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:57:03.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary2 2024-09-17T23:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary2': 2024-09-17T23:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 15417512aea4 2024-09-17T23:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.15417512aea4 2024-09-17T23:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:57:01 2024 2024-09-17T23:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:57:01 2024 2024-09-17T23:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:57:01 2024 2024-09-17T23:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: be7259a5-f0ba-4f85-b82a-ee2c6fa0005e 2024-09-17T23:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:57:03.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:57:03.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-17T23:57:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd1 2024-09-17T23:57:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:57:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd1 2024-09-17T23:57:03.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:57:03.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:57:03.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:57:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 8c0cfcfc-783c-4d51-b521-8a498ca29b5a 2024-09-17T23:57:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:57:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:57:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:57:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:57:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:57:03.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:57:03.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:03.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary2 2024-09-17T23:57:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd1 test-primary2 2024-09-17T23:57:03.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:57:03.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary2 2024-09-17T23:57:03.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary2 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary2 2024-09-17T23:57:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:57:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary2 2024-09-17T23:57:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-17T23:57:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary2 2024-09-17T23:57:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary3 2024-09-17T23:57:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary3 snapshot 10G 2024-09-17T23:57:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary3 10G 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary3 2024-09-17T23:57:03.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:57:03.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:03.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:57:03.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:57:03.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary3 snapshot 2024-09-17T23:57:03.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:03.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:03.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:03.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:03.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary3 snapshot 2024-09-17T23:57:03.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:57:03.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary2/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary2' 2024-09-17T23:57:04.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 336 2024-09-17T23:57:04.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:57:04.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:04.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary3 2024-09-17T23:57:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary3': 2024-09-17T23:57:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:57:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:57:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:57:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 15516f86fe77 2024-09-17T23:57:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.15516f86fe77 2024-09-17T23:57:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:57:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:57:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:57:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:57:04.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:57:03 2024 2024-09-17T23:57:04.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:57:03 2024 2024-09-17T23:57:04.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:57:03 2024 2024-09-17T23:57:04.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:57:04.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:57:04.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 485ee3af-7702-462b-9378-2e2be259b434 2024-09-17T23:57:04.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:57:04.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:57:04.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-17T23:57:04.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd2 2024-09-17T23:57:04.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:57:04.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd2 2024-09-17T23:57:04.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:57:04.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:57:04.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:57:04.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 601797bb-bc10-4670-84a4-5501d4356220 2024-09-17T23:57:04.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:57:04.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:57:04.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:04.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:57:04.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:57:04.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:57:04.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:57:04.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:04.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary3 2024-09-17T23:57:04.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd2 test-primary3 2024-09-17T23:57:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:57:04.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary3 2024-09-17T23:57:04.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:57:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary3 2024-09-17T23:57:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:57:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:57:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-17T23:57:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary4 2024-09-17T23:57:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary4 snapshot 10G 2024-09-17T23:57:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary3 2024-09-17T23:57:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary4 10G 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary3 2024-09-17T23:57:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:57:04.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:57:04.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:04.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary4 2024-09-17T23:57:04.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary3 2024-09-17T23:57:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:57:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:04.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:57:04.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:57:04.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:04.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:57:04.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:57:04.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary4 snapshot 2024-09-17T23:57:04.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:04.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:04.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:04.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:04.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary4 snapshot 2024-09-17T23:57:04.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:57:04.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary3/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary3' 2024-09-17T23:57:05.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:05.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:05.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:05.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:05.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:05.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:57:05.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 338 2024-09-17T23:57:05.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:57:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:05.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary4 2024-09-17T23:57:05.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary4': 2024-09-17T23:57:05.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1561efa208c0 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1561efa208c0 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:57:04 2024 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:57:04 2024 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:57:04 2024 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:57:05.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:57:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 3a58ab54-d559-485b-ad84-79ecd4d5dae7 2024-09-17T23:57:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:57:05.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:57:05.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-17T23:57:05.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd3 2024-09-17T23:57:05.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:57:05.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd3 2024-09-17T23:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:57:05.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:57:05.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:57:05.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 0ee0ab00-e613-4917-b8ad-3d890087929d 2024-09-17T23:57:05.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:57:05.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:57:05.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:05.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:57:05.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:57:05.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:57:05.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:57:05.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:05.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary4 2024-09-17T23:57:05.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd3 test-primary4 2024-09-17T23:57:05.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:57:05.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary4 2024-09-17T23:57:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:57:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary4 2024-09-17T23:57:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:57:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:57:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary4 2024-09-17T23:57:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-17T23:57:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary5 2024-09-17T23:57:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary5 snapshot 10G 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary4 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary4 2024-09-17T23:57:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary5 10G 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:05.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary5 2024-09-17T23:57:05.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:57:05.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:05.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:57:05.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:57:05.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary5 snapshot 2024-09-17T23:57:05.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:05.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:05.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:05.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:05.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary5 snapshot 2024-09-17T23:57:05.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:57:05.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary4/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary4' 2024-09-17T23:57:06.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 341 2024-09-17T23:57:06.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:06.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:57:06.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary5 2024-09-17T23:57:06.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary5': 2024-09-17T23:57:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:57:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:57:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:57:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1574d8b966e5 2024-09-17T23:57:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1574d8b966e5 2024-09-17T23:57:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:57:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:57:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:57:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:57:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:57:05 2024 2024-09-17T23:57:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:57:05 2024 2024-09-17T23:57:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:57:05 2024 2024-09-17T23:57:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:57:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:57:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 9d12657b-7828-4919-923a-59e2a9847f2c 2024-09-17T23:57:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:57:06.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:57:06.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-17T23:57:06.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd4 2024-09-17T23:57:06.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:57:06.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd4 2024-09-17T23:57:06.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:57:06.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:57:06.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:57:06.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: b979b442-c044-4eaf-82f0-e1e282ad0d43 2024-09-17T23:57:06.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:57:06.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:57:06.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:06.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:57:06.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:57:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:57:06.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:57:06.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:06.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary5 2024-09-17T23:57:06.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd4 test-primary5 2024-09-17T23:57:06.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:57:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary5 2024-09-17T23:57:06.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 340 2024-09-17T23:57:06.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:57:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary5 2024-09-17T23:57:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:57:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary5 2024-09-17T23:57:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:57:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-17T23:57:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary6 2024-09-17T23:57:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:57:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary6 snapshot 10G 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary5 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary5 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:06.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:06.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:57:06.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:06.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary6 10G 2024-09-17T23:57:06.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:06.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:06.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:06.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:06.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:06.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:06.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:57:06.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:57:06.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:57:06.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:06.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary6 2024-09-17T23:57:06.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:07.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:57:07.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:57:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:57:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary6 snapshot 2024-09-17T23:57:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary6 snapshot 2024-09-17T23:57:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:57:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:07.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:57:07.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:57:07.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary5/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary5' 2024-09-17T23:57:08.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:08.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:57:08.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:08.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:08.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:08.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:57:08.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 343 2024-09-17T23:57:08.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:57:08.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:08.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary6 2024-09-17T23:57:08.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary6': 2024-09-17T23:57:08.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:57:08.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:57:08.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:57:08.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 15804a92ef3b 2024-09-17T23:57:08.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.15804a92ef3b 2024-09-17T23:57:08.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:57:08.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:57:08.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:57:08.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:57:08.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:57:07 2024 2024-09-17T23:57:08.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:57:07 2024 2024-09-17T23:57:08.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:57:07 2024 2024-09-17T23:57:08.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:57:08.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:57:08.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 631249db-2f37-4574-b5c1-3ddb41b94859 2024-09-17T23:57:08.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:57:08.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:57:08.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-17T23:57:08.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd5 2024-09-17T23:57:08.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:57:08.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd5 2024-09-17T23:57:08.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:57:08.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:57:08.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:57:08.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 9c3fb7cb-e382-479c-966d-c160c00e84e2 2024-09-17T23:57:08.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:57:08.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:57:08.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:08.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:57:08.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:57:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:57:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:57:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary6 2024-09-17T23:57:08.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd5 test-primary6 2024-09-17T23:57:08.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:57:08.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary6 2024-09-17T23:57:08.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:57:08.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary6 2024-09-17T23:57:08.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary6 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary7 2024-09-17T23:57:08.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary7 snapshot 10G 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary6 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary6 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:08.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary7 10G 2024-09-17T23:57:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:57:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:57:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:57:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary7 2024-09-17T23:57:08.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:57:08.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:08.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:57:08.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:57:08.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary7 snapshot 2024-09-17T23:57:08.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:08.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:08.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:08.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:08.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary7 snapshot 2024-09-17T23:57:08.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:57:08.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary6/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary6' 2024-09-17T23:57:09.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 345 2024-09-17T23:57:09.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 347 2024-09-17T23:57:09.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:09.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:09.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:57:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:57:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:09.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:57:09.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary7 2024-09-17T23:57:09.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary7': 2024-09-17T23:57:09.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:57:09.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:57:09.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:57:09.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1595ecefcf0a 2024-09-17T23:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1595ecefcf0a 2024-09-17T23:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:57:08 2024 2024-09-17T23:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:57:08 2024 2024-09-17T23:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:57:08 2024 2024-09-17T23:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 40e08d00-c76c-475b-a9ba-1e4c6459df5a 2024-09-17T23:57:09.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:57:09.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:57:09.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-17T23:57:09.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd6 2024-09-17T23:57:09.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:57:09.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd6 2024-09-17T23:57:09.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:57:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:57:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:57:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 8268ea0e-5d3b-4179-ab63-427cc5144a6c 2024-09-17T23:57:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:57:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:57:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:57:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:57:09.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:09.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary7 2024-09-17T23:57:09.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd6 test-primary7 2024-09-17T23:57:09.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:57:09.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary7 2024-09-17T23:57:09.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:57:09.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary7 2024-09-17T23:57:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:57:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:57:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-17T23:57:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary8 2024-09-17T23:57:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:57:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary7 2024-09-17T23:57:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary8 snapshot 10G 2024-09-17T23:57:09.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary7 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:09.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary8 10G 2024-09-17T23:57:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary7 2024-09-17T23:57:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:57:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:57:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:57:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:09.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary8 2024-09-17T23:57:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:57:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:57:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:57:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary8 snapshot 2024-09-17T23:57:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary8 snapshot 2024-09-17T23:57:09.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:57:09.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary7/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary7' 2024-09-17T23:57:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:09.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:57:10.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 346 2024-09-17T23:57:10.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:10.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 350 2024-09-17T23:57:10.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:10.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:57:10.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary8 2024-09-17T23:57:10.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary8': 2024-09-17T23:57:10.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:57:10.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 15ab370dd298 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.15ab370dd298 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:57:09 2024 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:57:09 2024 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:57:09 2024 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 11e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7e 2024-09-17T23:57:10.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:57:10.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:57:10.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-17T23:57:10.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd7 2024-09-17T23:57:10.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:57:10.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd7 2024-09-17T23:57:10.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:57:10.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:57:10.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:57:10.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 13848b0a-f463-4377-863d-ee62ed1e8e4c 2024-09-17T23:57:10.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:57:10.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:57:10.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:10.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:57:10.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:57:10.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:57:10.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:57:10.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:10.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary8 2024-09-17T23:57:10.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd7 test-primary8 2024-09-17T23:57:10.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:57:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary8 2024-09-17T23:57:10.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:57:10.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary8 2024-09-17T23:57:10.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:10.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:10.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:10.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:57:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:57:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:57:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:57:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary8 2024-09-17T23:57:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-17T23:57:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary9 2024-09-17T23:57:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary9 snapshot 10G 2024-09-17T23:57:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary8 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary8 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary9 10G 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:10.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:10.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:10.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:10.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:57:10.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:57:10.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:57:10.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:10.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary9 2024-09-17T23:57:10.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:57:10.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:10.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:57:10.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:57:10.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary9 snapshot 2024-09-17T23:57:10.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:10.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:10.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:10.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:10.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary9 snapshot 2024-09-17T23:57:10.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:57:10.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary8/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary8' 2024-09-17T23:57:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 349 2024-09-17T23:57:11.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:11.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 353 2024-09-17T23:57:11.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:11.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:57:11.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:11.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:57:11.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:11.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:11.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:11.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:57:11.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary9 2024-09-17T23:57:11.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary9': 2024-09-17T23:57:11.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 15be23fe24 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.15be23fe24 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:57:10 2024 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:57:10 2024 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:57:10 2024 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 91a54e9f-7ef4-41db-964a-c84ca41a65ce 2024-09-17T23:57:11.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:57:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:57:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-17T23:57:11.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd8 2024-09-17T23:57:11.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:57:11.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd8 2024-09-17T23:57:11.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:57:11.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:57:11.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:57:11.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 534804c5-15f8-41dc-8fad-5334f83ad9e9 2024-09-17T23:57:11.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:57:11.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:57:11.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:11.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:57:11.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:57:11.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:57:11.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:57:11.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:11.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary9 2024-09-17T23:57:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd8 test-primary9 2024-09-17T23:57:11.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:57:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary9 2024-09-17T23:57:11.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:57:11.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary9 2024-09-17T23:57:11.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary10 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary10 snapshot 10G 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary9 2024-09-17T23:57:11.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary10 10G 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary9 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:11.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:11.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:11.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:11.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary9 2024-09-17T23:57:11.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-17T23:57:11.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-17T23:57:11.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-17T23:57:11.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-17T23:57:11.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary10 2024-09-17T23:57:11.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 352 2024-09-17T23:57:11.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-17T23:57:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-17T23:57:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-17T23:57:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary10 snapshot 2024-09-17T23:57:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-17T23:57:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary10 snapshot 2024-09-17T23:57:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:57:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary9/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary9' 2024-09-17T23:57:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:57:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:12.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:57:12.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:12.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:57:12.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:12.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:12.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:12.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:57:12.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 355 2024-09-17T23:57:12.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:13.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 356 2024-09-17T23:57:13.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-17T23:57:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:13.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary10 2024-09-17T23:57:13.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:13.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:57:13.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:13.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:13.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:13.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:57:13.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:13.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:57:13.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:13.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:13.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:13.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:57:13.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary10': 2024-09-17T23:57:13.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-17T23:57:13.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-17T23:57:13.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-17T23:57:13.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 15d055ba946c 2024-09-17T23:57:13.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.15d055ba946c 2024-09-17T23:57:13.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-17T23:57:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-17T23:57:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-17T23:57:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-17T23:57:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Tue Sep 17 23:57:11 2024 2024-09-17T23:57:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Tue Sep 17 23:57:11 2024 2024-09-17T23:57:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Tue Sep 17 23:57:11 2024 2024-09-17T23:57:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-17T23:57:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-17T23:57:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 527eda38-0231-4aa6-80fd-b8db0934cf7b 2024-09-17T23:57:13.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-17T23:57:13.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-17T23:57:13.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-17T23:57:13.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd9 2024-09-17T23:57:13.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-17T23:57:13.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd9 2024-09-17T23:57:13.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-17T23:57:13.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-17T23:57:13.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-17T23:57:13.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 13f84d0d-b5a1-4a1c-a9cb-4c8504148df1 2024-09-17T23:57:13.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-17T23:57:13.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-17T23:57:13.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:13.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-17T23:57:13.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-17T23:57:13.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-17T23:57:13.646 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:57:13.641+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0b663790 2024-09-17T23:57:13.646 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:57:13.641+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0babea90 2024-09-17T23:57:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-17T23:57:13.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-17T23:57:13.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary10 2024-09-17T23:57:13.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd9 test-primary10 2024-09-17T23:57:13.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-17T23:57:13.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary10 2024-09-17T23:57:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-17T23:57:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary10 2024-09-17T23:57:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-17T23:57:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:57:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 41319 2024-09-17T23:57:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:57:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:13.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:57:13.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary10 2024-09-17T23:57:13.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary10 2024-09-17T23:57:13.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary10 2024-09-17T23:57:14.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 359 2024-09-17T23:57:14.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 358 2024-09-17T23:57:14.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:14.024 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:57:14.017+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0b1d6c30 2024-09-17T23:57:14.024 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:57:14.017+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0babea90 2024-09-17T23:57:14.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:14.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-17T23:57:14.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary10/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary10' 2024-09-17T23:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:57:14.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:14.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:57:14.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:14.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:14.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:14.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:57:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:57:14.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 361 2024-09-17T23:57:15.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:15.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 360 2024-09-17T23:57:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:15.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 362 2024-09-17T23:57:15.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 363 2024-09-17T23:57:16.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:16.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:57:16.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:16.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:16.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:16.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:57:16.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 364 2024-09-17T23:57:16.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:16.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:16.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:57:16.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:16.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:16.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:16.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:57:17.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 365 2024-09-17T23:57:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:57:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:17.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:17.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:17.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:57:17.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:57:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:17.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:17.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:57:18.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:18.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:57:18.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:18.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:18.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:18.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:57:18.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 366 2024-09-17T23:57:18.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:18.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:57:18.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:18.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:18.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:18.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:57:18.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 367 2024-09-17T23:57:18.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:18.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:18.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:57:18.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:18.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:18.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:18.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:57:18.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:19.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:19.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:57:19.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:19.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:19.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:19.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:57:19.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 368 2024-09-17T23:57:19.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:19.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:57:19.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:19.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:19.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:19.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:57:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:19.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 369 2024-09-17T23:57:19.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:20.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:20.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:57:20.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 372 2024-09-17T23:57:20.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 371 2024-09-17T23:57:20.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:20.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 370 2024-09-17T23:57:20.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 373 2024-09-17T23:57:21.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 374 2024-09-17T23:57:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:57:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:57:21.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:21.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:57:21.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:21.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:21.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:21.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:57:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 375 2024-09-17T23:57:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:22.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:22.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:57:22.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:22.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:22.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:22.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:57:22.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:22.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:57:22.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:22.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:22.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:22.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:57:23.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 376 2024-09-17T23:57:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:57:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:57:23.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:23.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 377 2024-09-17T23:57:23.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:23.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:57:23.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:23.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:23.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:23.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:57:23.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:23.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:23.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:57:23.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:23.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:23.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:23.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:57:24.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:24.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:57:24.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:24.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:24.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:24.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:57:24.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 379 2024-09-17T23:57:24.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 378 2024-09-17T23:57:24.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:24.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:24.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:24.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:24.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:24.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:24.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:24.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:57:25.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 382 2024-09-17T23:57:25.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 380 2024-09-17T23:57:25.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 381 2024-09-17T23:57:25.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:25.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:25.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:26.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:26.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:57:26.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:26.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:26.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:26.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:57:26.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 384 2024-09-17T23:57:26.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 383 2024-09-17T23:57:26.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:26.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:26.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:26.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:57:26.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:26.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:26.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:26.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:57:26.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 385 2024-09-17T23:57:27.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:27.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:27.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:57:27.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:27.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:27.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:27.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:57:27.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:27.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:57:27.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:27.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:27.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:27.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:57:28.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 386 2024-09-17T23:57:28.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:28.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:57:28.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:28.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:28.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:28.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:57:28.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:28.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:57:28.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:28.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:28.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:28.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:57:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:57:28.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:28.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:28.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:28.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:57:28.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:28.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 387 2024-09-17T23:57:28.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 388 2024-09-17T23:57:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:57:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:57:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:57:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:57:29.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:29.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 389 2024-09-17T23:57:29.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:57:30.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 392 2024-09-17T23:57:30.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 390 2024-09-17T23:57:30.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 391 2024-09-17T23:57:30.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:30.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:30.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:31.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 394 2024-09-17T23:57:31.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:31.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:57:31.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:31.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:31.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:31.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:57:31.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 393 2024-09-17T23:57:31.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:57:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:31.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:57:31.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:31.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 395 2024-09-17T23:57:32.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:32.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:32.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:57:32.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:32.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:32.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:32.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:57:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:32.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:57:32.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:32.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:32.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:32.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:57:33.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 397 2024-09-17T23:57:33.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 396 2024-09-17T23:57:33.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:57:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:57:33.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:33.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:57:33.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:33.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:33.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:33.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:57:33.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:33.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:57:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:33.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:57:34.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:34.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:57:34.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:34.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:34.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:34.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:57:34.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:34.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:57:34.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:34.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:34.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:34.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:57:34.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 398 2024-09-17T23:57:34.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 399 2024-09-17T23:57:34.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:34.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:35.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:35.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:35.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:35.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:35.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:35.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:57:35.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 402 2024-09-17T23:57:35.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 401 2024-09-17T23:57:35.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 400 2024-09-17T23:57:35.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:35.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:35.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:57:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:36.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:57:36.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:36.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:57:36.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:36.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:36.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:36.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:57:36.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 404 2024-09-17T23:57:36.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 403 2024-09-17T23:57:36.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:36.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:37.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 405 2024-09-17T23:57:37.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:37.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:37.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:57:37.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:37.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:37.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:37.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:57:37.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:37.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:57:37.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:37.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:37.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:37.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:57:38.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 407 2024-09-17T23:57:38.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 406 2024-09-17T23:57:38.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:38.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:57:38.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:38.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:38.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:38.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:57:38.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:38.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:57:38.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:38.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:38.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:38.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:38.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:57:38.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:38.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:38.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:57:38.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:38.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:38.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:38.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:57:39.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 408 2024-09-17T23:57:39.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:39.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:39.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:57:39.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:39.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:39.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:39.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:57:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:57:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:57:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 409 2024-09-17T23:57:39.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:40.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:40.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:40.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:57:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 412 2024-09-17T23:57:40.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 410 2024-09-17T23:57:40.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 411 2024-09-17T23:57:40.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:40.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:40.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 413 2024-09-17T23:57:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:57:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:57:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:57:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:41.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:41.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:41.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:57:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:41.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 414 2024-09-17T23:57:41.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:41.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 415 2024-09-17T23:57:41.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:57:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:57:42.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:42.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:57:42.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:42.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:42.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:42.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:57:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 416 2024-09-17T23:57:42.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 417 2024-09-17T23:57:43.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:43.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:57:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:57:43.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:43.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:57:43.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:43.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:43.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:43.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:57:43.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:43.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:57:43.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:43.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:43.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:43.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:57:44.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 418 2024-09-17T23:57:44.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:44.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 419 2024-09-17T23:57:44.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:44.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:44.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:57:44.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:44.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:44.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:44.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:57:44.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:44.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:57:44.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:44.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:44.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:44.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:57:44.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:45.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:45.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:45.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:45.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:45.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:57:45.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 422 2024-09-17T23:57:45.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 421 2024-09-17T23:57:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 420 2024-09-17T23:57:45.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:45.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:45.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:45.868 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:57:45.864+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0978f5f0 2024-09-17T23:57:45.868 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:57:45.864+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0b1d6c30 2024-09-17T23:57:45.868 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:57:45.864+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0b663790 2024-09-17T23:57:45.868 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:57:45.864+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0babea90 2024-09-17T23:57:46.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 423 2024-09-17T23:57:46.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:46.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:46.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:57:46.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:57:46.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:46.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:46.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:46.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:46.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:46.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:46.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:57:46.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:57:46.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 424 2024-09-17T23:57:46.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:46.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:46.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 425 2024-09-17T23:57:47.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:47.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:47.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:57:47.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:47.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:47.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:47.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:57:47.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:47.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:57:47.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:47.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:47.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:47.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:57:48.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 426 2024-09-17T23:57:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:57:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:57:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:48.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:57:48.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:48.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:48.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:48.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:57:48.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:48.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:48.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:57:48.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:48.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:48.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:48.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:57:48.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 427 2024-09-17T23:57:48.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:49.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:57:49.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:49.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:49.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:49.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:57:49.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:49.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:57:49.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:49.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:49.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:49.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:57:49.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 429 2024-09-17T23:57:49.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 428 2024-09-17T23:57:49.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:49.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:50.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:50.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:50.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:50.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:50.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:50.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:57:50.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 431 2024-09-17T23:57:50.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 430 2024-09-17T23:57:50.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 432 2024-09-17T23:57:50.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:50.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:50.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:51.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:51.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:57:51.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:51.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:51.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:51.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:57:51.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 434 2024-09-17T23:57:51.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 433 2024-09-17T23:57:51.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:51.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:51.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:57:51.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:51.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:51.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:51.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:57:51.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:52.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 435 2024-09-17T23:57:52.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:52.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:52.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:57:52.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:52.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:52.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:52.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:57:52.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:52.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:57:52.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:52.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:52.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:52.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:57:53.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 436 2024-09-17T23:57:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:57:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:57:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:57:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:57:53.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 437 2024-09-17T23:57:53.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:53.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:57:53.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:53.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:53.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:53.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:57:53.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:53.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:54.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:54.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:57:54.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:54.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:54.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:54.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:57:54.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:54.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:57:54.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:54.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:54.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:54.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:57:54.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 438 2024-09-17T23:57:54.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 439 2024-09-17T23:57:54.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:54.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:55.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:55.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:55.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:55.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:55.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:55.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:57:55.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 442 2024-09-17T23:57:55.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 440 2024-09-17T23:57:55.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 441 2024-09-17T23:57:55.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:55.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:55.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:56.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:56.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:57:56.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:56.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:56.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:57:56.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:57:56.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:56.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:57:56.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:56.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:56.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:57:56.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:57:56.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 444 2024-09-17T23:57:56.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 443 2024-09-17T23:57:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:56.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 445 2024-09-17T23:57:56.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:57.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:57.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:57:57.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:57.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:57.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:57:57.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:57:57.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:57.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:57:57.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:57.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:57.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:57:57.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:57:58.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:58.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:57:58.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:58.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:57:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:57:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:57:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:57:58.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:57:58.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:58.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:57:58.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:58.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:58.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:57:58.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:57:58.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 446 2024-09-17T23:57:58.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:58.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 447 2024-09-17T23:57:58.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:59.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 448 2024-09-17T23:57:59.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 449 2024-09-17T23:57:59.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:59.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:57:59.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:59.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:59.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:57:59.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:57:59.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:59.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:57:59.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:59.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:59.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:57:59.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:57:59.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:59.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:57:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:57:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:57:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:57:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:57:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:57:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:58:00.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 452 2024-09-17T23:58:00.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 450 2024-09-17T23:58:00.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:00.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 451 2024-09-17T23:58:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:01.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 453 2024-09-17T23:58:01.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 454 2024-09-17T23:58:01.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:01.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:58:01.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:01.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:01.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:58:01.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:58:01.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:01.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:01.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:01.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:58:01.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:01.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:01.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:58:01.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:58:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 455 2024-09-17T23:58:02.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:58:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:58:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:58:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:58:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:58:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:58:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 457 2024-09-17T23:58:03.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 456 2024-09-17T23:58:03.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:03.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:03.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:03.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:58:03.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:03.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:03.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:58:03.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:58:03.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:03.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:58:03.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:03.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:03.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:03.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:58:03.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:58:03.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:58:03.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:03.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:03.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:58:03.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:58:04.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 458 2024-09-17T23:58:04.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 459 2024-09-17T23:58:04.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:04.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:58:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:04.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:58:04.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:58:04.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:04.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:58:04.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:58:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:58:05.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 462 2024-09-17T23:58:05.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 461 2024-09-17T23:58:05.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:05.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:58:05.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:05.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:05.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:58:05.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:58:05.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:05.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 460 2024-09-17T23:58:05.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:06.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:06.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:58:06.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:06.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:06.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:58:06.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:58:06.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:06.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:58:06.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:06.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:06.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:58:06.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:58:06.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 463 2024-09-17T23:58:06.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 464 2024-09-17T23:58:06.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:06.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:07.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 465 2024-09-17T23:58:07.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:07.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:07.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:58:07.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:07.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:07.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:58:07.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:58:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:58:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:58:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:58:08.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:08.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:58:08.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:08.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:08.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:58:08.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:58:08.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:58:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:58:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:58:08.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 467 2024-09-17T23:58:08.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 466 2024-09-17T23:58:08.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:08.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:58:08.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:08.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:08.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:58:08.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:58:08.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:08.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:09.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:09.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:58:09.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:09.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:09.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:58:09.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:58:09.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:09.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:58:09.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:09.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:09.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:58:09.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:58:09.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 468 2024-09-17T23:58:09.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 469 2024-09-17T23:58:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:09.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:10.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:10.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:58:10.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:10.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:10.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:58:10.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:58:10.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 470 2024-09-17T23:58:10.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 472 2024-09-17T23:58:10.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:10.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 471 2024-09-17T23:58:10.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:10.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:11.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:11.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:58:11.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:11.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:11.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:58:11.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:58:11.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:11.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:58:11.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:11.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:11.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:58:11.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:58:11.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 473 2024-09-17T23:58:11.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 474 2024-09-17T23:58:11.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:11.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 475 2024-09-17T23:58:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:12.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:12.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:58:12.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:12.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:12.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:58:12.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:58:12.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:12.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:58:12.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:12.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:12.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:58:12.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:58:13.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:13.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:58:13.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:13.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:13.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:58:13.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:58:13.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:13.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:58:13.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:13.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:13.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:58:13.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:58:13.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:13.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:58:13.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:13.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:13.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:58:13.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:58:13.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 476 2024-09-17T23:58:13.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 477 2024-09-17T23:58:13.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:13.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:14.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 478 2024-09-17T23:58:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:58:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:58:14.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:58:14.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:14.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 479 2024-09-17T23:58:14.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:14.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:58:14.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:14.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:14.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:58:14.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:58:14.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:15.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:15.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:58:15.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:15.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:15.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:58:15.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:58:15.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 481 2024-09-17T23:58:15.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:15.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 480 2024-09-17T23:58:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 482 2024-09-17T23:58:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:15.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:16.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 484 2024-09-17T23:58:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:58:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:58:16.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:58:16.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 483 2024-09-17T23:58:16.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:58:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:58:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:58:16.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:16.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:17.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 485 2024-09-17T23:58:17.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:58:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:58:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:58:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:58:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:58:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:58:18.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:18.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:58:18.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:18.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:18.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:58:18.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:58:18.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 487 2024-09-17T23:58:18.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 486 2024-09-17T23:58:18.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:18.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:58:18.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:18.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:18.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:58:18.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:58:18.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:18.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:18.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:18.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:58:18.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:18.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:18.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:58:18.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:58:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:58:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:58:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:58:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:58:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:58:19.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:58:19.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 488 2024-09-17T23:58:19.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:19.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 489 2024-09-17T23:58:19.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:20.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:20.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:58:20.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:20.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:20.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:58:20.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:58:20.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 490 2024-09-17T23:58:20.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 491 2024-09-17T23:58:20.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 492 2024-09-17T23:58:20.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:20.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:20.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:21.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 493 2024-09-17T23:58:21.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:21.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:58:21.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:21.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:21.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:58:21.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:58:21.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:21.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:58:21.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:21.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:21.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:58:21.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:58:21.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 494 2024-09-17T23:58:21.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:21.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:22.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 495 2024-09-17T23:58:22.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:58:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:58:22.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:58:22.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:22.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:58:22.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:22.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:22.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:58:22.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:58:23.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 497 2024-09-17T23:58:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 496 2024-09-17T23:58:23.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:23.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:23.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:58:23.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:23.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:23.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:58:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:58:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:58:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:58:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:58:23.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:23.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:58:23.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:23.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:23.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:58:23.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:24.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 499 2024-09-17T23:58:24.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:24.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:58:24.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:24.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:24.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:58:24.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:58:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 498 2024-09-17T23:58:24.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:58:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:58:24.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:58:24.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:25.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:25.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:58:25.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:25.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:25.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:58:25.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:58:25.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 502 2024-09-17T23:58:25.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 501 2024-09-17T23:58:25.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 500 2024-09-17T23:58:25.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:25.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:26.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 503 2024-09-17T23:58:26.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 504 2024-09-17T23:58:26.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:26.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:26.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:26.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:58:26.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:26.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:26.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:58:26.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:58:26.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:26.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:58:26.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:26.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:26.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:58:26.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:58:27.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 505 2024-09-17T23:58:27.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:27.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:27.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:58:27.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:27.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:27.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:58:27.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:58:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:58:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:27.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:27.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:58:27.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:58:28.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 506 2024-09-17T23:58:28.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:28.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 507 2024-09-17T23:58:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:58:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:58:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:58:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:58:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:28.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:58:28.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:58:28.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:28.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:58:28.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:28.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:28.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:58:28.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:58:29.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 509 2024-09-17T23:58:29.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 508 2024-09-17T23:58:29.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:29.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:29.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:29.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:58:29.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:29.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:29.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:58:29.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:58:29.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:29.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:58:29.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:29.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:29.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:58:29.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:58:30.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 510 2024-09-17T23:58:30.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:30.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:58:30.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:30.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:30.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:58:30.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:58:30.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:30.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 511 2024-09-17T23:58:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 512 2024-09-17T23:58:30.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:31.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 513 2024-09-17T23:58:31.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:31.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:58:31.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:31.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:31.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:58:31.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:58:31.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:31.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 514 2024-09-17T23:58:31.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:31.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:58:31.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:31.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:31.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:58:31.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:58:31.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:32.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 515 2024-09-17T23:58:32.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:58:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:58:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:58:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:58:32.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:58:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:58:33.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:33.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:58:33.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:33.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:33.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:58:33.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:58:33.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:58:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 516 2024-09-17T23:58:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:58:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:58:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:58:33.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:33.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 517 2024-09-17T23:58:33.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:34.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:34.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:58:34.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:34.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:34.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:58:34.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:58:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 518 2024-09-17T23:58:34.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:34.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:34.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:58:34.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:34.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:34.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:58:34.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:58:34.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 519 2024-09-17T23:58:34.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:35.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:35.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:58:35.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:35.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:35.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:58:35.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:58:35.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 520 2024-09-17T23:58:35.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 521 2024-09-17T23:58:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 522 2024-09-17T23:58:35.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:35.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:35.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:58:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:36.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:58:36.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:58:36.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 524 2024-09-17T23:58:36.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 523 2024-09-17T23:58:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:58:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:58:36.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:58:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:36.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:37.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 525 2024-09-17T23:58:37.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:58:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:58:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:58:37.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:58:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:58:37.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:58:38.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 527 2024-09-17T23:58:38.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 526 2024-09-17T23:58:38.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:38.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:58:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:58:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:58:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:38.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:38.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:58:38.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:38.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:38.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:58:38.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:58:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:58:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:38.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:38.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:58:38.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:58:39.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 528 2024-09-17T23:58:39.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:39.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:58:39.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:39.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:39.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:58:39.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:58:39.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:39.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:58:39.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:39.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:39.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:58:39.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:58:39.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 529 2024-09-17T23:58:39.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:39.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:40.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:40.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:58:40.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:40.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:40.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:58:40.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:58:40.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 532 2024-09-17T23:58:40.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 530 2024-09-17T23:58:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:40.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:40.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 531 2024-09-17T23:58:40.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 533 2024-09-17T23:58:41.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:41.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:58:41.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:41.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:41.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:58:41.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:58:41.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:58:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:58:41.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:58:41.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:41.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 534 2024-09-17T23:58:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:42.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 535 2024-09-17T23:58:42.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:42.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:42.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:58:42.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:42.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:42.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:58:42.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:58:42.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:42.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:58:42.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:42.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:42.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:58:42.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:58:43.119 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-17T23:58:43.118+0000 7f849c00d640 -1 reset not still connected to 0x5629c733bc70 2024-09-17T23:58:43.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 537 2024-09-17T23:58:43.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:43.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:58:43.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:43.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:43.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:58:43.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:58:43.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 536 2024-09-17T23:58:43.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:43.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:58:43.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:43.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:43.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:58:43.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:58:43.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:43.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:58:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:58:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:58:44.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 539 2024-09-17T23:58:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:44.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:58:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:58:44.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:58:44.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 538 2024-09-17T23:58:44.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:58:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:58:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:58:44.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:58:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:58:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:58:45.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 542 2024-09-17T23:58:45.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:45.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 540 2024-09-17T23:58:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 541 2024-09-17T23:58:45.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:45.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:46.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 544 2024-09-17T23:58:46.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 543 2024-09-17T23:58:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:58:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:46.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:46.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:58:46.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:58:46.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:46.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:58:46.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:46.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:46.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:58:46.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:58:46.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:46.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:47.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 545 2024-09-17T23:58:47.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:47.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:47.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:58:47.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:47.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:47.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:58:47.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:58:47.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:47.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:58:47.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:47.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:47.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:58:47.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:58:48.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 546 2024-09-17T23:58:48.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 547 2024-09-17T23:58:48.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:58:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:58:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:58:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:48.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:58:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:58:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:58:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:58:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:58:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:58:49.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 549 2024-09-17T23:58:49.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:49.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:58:49.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:49.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:49.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:58:49.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:58:49.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:49.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:58:49.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:49.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:49.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:58:49.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:58:49.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:49.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 548 2024-09-17T23:58:49.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:50.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:50.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:58:50.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:50.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:50.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:58:50.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:58:50.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 550 2024-09-17T23:58:50.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 552 2024-09-17T23:58:50.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 551 2024-09-17T23:58:50.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:51.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:51.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:58:51.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:51.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:51.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:58:51.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:58:51.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:51.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:58:51.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:51.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:51.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:58:51.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:58:51.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 554 2024-09-17T23:58:51.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 553 2024-09-17T23:58:51.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:51.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:52.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 555 2024-09-17T23:58:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:52.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:52.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:58:52.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:52.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:52.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:58:52.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:58:52.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:52.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:58:52.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:52.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:52.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:58:52.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:58:53.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 556 2024-09-17T23:58:53.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 557 2024-09-17T23:58:53.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:53.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:53.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:53.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:58:53.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:53.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:53.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:58:53.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:58:53.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:53.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:58:53.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:53.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:53.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:58:53.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:58:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:58:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:58:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:58:54.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 559 2024-09-17T23:58:54.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 558 2024-09-17T23:58:54.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:54.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:54.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:54.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:58:54.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:54.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:54.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:58:54.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:58:54.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:54.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:58:54.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:54.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:54.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:58:54.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:58:55.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 560 2024-09-17T23:58:55.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:55.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:58:55.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:55.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:55.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:58:55.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:58:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 562 2024-09-17T23:58:55.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 561 2024-09-17T23:58:55.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:55.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:58:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:58:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:56.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:58:56.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:58:56.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:56.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:56.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:58:56.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:58:56.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 563 2024-09-17T23:58:56.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 564 2024-09-17T23:58:56.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:56.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:57.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 565 2024-09-17T23:58:57.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:57.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:57.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:58:57.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:57.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:57.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:58:57.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:58:57.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:57.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:58:57.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:57.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:57.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:58:57.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:58:58.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:58.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:58:58.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:58.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:58.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:58:58.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:58:58.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 566 2024-09-17T23:58:58.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:58.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:58:58.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:58.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:58.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:58:58.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:58:58.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 567 2024-09-17T23:58:58.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:58.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:58:58.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:58.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:58.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:58:58.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:58:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:58.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:59.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:59.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:58:59.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:59.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:59.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:58:59.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:58:59.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 568 2024-09-17T23:58:59.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:58:59.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:58:59.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:58:59.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:58:59.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:58:59.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:58:59.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 569 2024-09-17T23:58:59.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:58:59.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:59:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:00.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:59:00.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:59:00.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 570 2024-09-17T23:59:00.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 572 2024-09-17T23:59:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 571 2024-09-17T23:59:00.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:00.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:00.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:00.792 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:59:00.786+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0b663790 2024-09-17T23:59:00.792 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-17T23:59:00.786+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0babea90 2024-09-17T23:59:01.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:01.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:59:01.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:01.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:01.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:59:01.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:59:01.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 573 2024-09-17T23:59:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:59:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:59:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:59:01.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 574 2024-09-17T23:59:01.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:01.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 575 2024-09-17T23:59:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:02.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:02.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:59:02.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:02.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:02.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:59:02.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:59:02.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:02.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:59:02.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:02.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:02.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:59:02.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:59:03.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 577 2024-09-17T23:59:03.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:03.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:59:03.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:03.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:03.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:59:03.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:59:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:59:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:59:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:59:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 576 2024-09-17T23:59:03.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:03.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:03.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:59:03.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:03.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:03.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:59:03.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:59:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 579 2024-09-17T23:59:04.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 578 2024-09-17T23:59:04.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:04.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:59:04.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:04.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:04.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:59:04.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:59:04.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:04.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:59:04.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:04.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:04.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:59:04.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:59:04.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:04.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:04.153 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-17T23:59:04.153+0000 7f849c00d640 -1 reset not still connected to 0x5629a8566410 2024-09-17T23:59:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:59:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:59:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:59:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 582 2024-09-17T23:59:05.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 581 2024-09-17T23:59:05.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:05.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 580 2024-09-17T23:59:05.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:05.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:06.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:06.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:59:06.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:06.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:06.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:59:06.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:59:06.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:06.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:59:06.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 584 2024-09-17T23:59:06.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:06.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:06.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:59:06.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:59:06.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 583 2024-09-17T23:59:06.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:07.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 585 2024-09-17T23:59:07.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:07.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:07.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:59:07.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:07.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:07.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:59:07.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:59:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:59:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:59:07.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:59:08.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 587 2024-09-17T23:59:08.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:08.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:59:08.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:08.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:08.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:59:08.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:59:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 586 2024-09-17T23:59:08.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:08.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:59:08.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:08.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:08.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:59:08.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:59:08.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:08.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:59:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:59:08.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:59:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:09.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:09.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:59:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:59:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:59:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:59:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:59:09.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:59:09.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 588 2024-09-17T23:59:09.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:09.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 589 2024-09-17T23:59:09.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:10.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:10.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:59:10.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:10.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:10.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:59:10.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:59:10.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 592 2024-09-17T23:59:10.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 590 2024-09-17T23:59:10.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 591 2024-09-17T23:59:10.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:10.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:11.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:11.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:59:11.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:11.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:11.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:59:11.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:59:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:59:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:59:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:59:11.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 593 2024-09-17T23:59:11.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 594 2024-09-17T23:59:11.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:11.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:12.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 595 2024-09-17T23:59:12.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:12.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:12.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:59:12.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:12.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:12.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:59:12.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:59:12.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:12.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:59:12.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:59:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:59:13.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:13.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:59:13.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:13.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:13.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:59:13.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:59:13.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:13.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:59:13.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:13.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:13.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:59:13.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:59:13.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:13.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:59:13.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:13.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:13.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:59:13.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:59:13.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 596 2024-09-17T23:59:13.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 597 2024-09-17T23:59:13.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:13.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:14.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 598 2024-09-17T23:59:14.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:14.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:59:14.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:14.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:14.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:59:14.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:59:14.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:14.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:59:14.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:14.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:14.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:59:14.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:59:14.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:14.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 599 2024-09-17T23:59:14.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:15.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:15.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:59:15.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:59:15.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 600 2024-09-17T23:59:15.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:15.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 601 2024-09-17T23:59:15.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 602 2024-09-17T23:59:15.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:15.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:16.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 603 2024-09-17T23:59:16.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 604 2024-09-17T23:59:16.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:16.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:16.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:59:16.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:16.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:16.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:59:16.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:59:16.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:16.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:59:16.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:16.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:16.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:59:16.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:59:16.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 605 2024-09-17T23:59:17.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:17.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:17.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:59:17.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:17.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:17.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:59:17.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:59:17.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:17.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:59:17.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:17.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:17.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:59:17.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:59:18.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 606 2024-09-17T23:59:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 607 2024-09-17T23:59:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:59:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:59:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:59:18.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:18.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:59:18.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:18.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:18.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:59:18.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:59:18.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:18.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:59:18.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:18.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:18.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:59:18.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 609 2024-09-17T23:59:19.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 608 2024-09-17T23:59:19.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:19.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:59:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:59:19.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:59:19.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:19.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:59:19.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:19.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:19.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:59:19.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:59:20.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:20.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:59:20.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:20.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:20.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:59:20.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:59:20.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 611 2024-09-17T23:59:20.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 612 2024-09-17T23:59:20.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 610 2024-09-17T23:59:20.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:20.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:20.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:21.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 613 2024-09-17T23:59:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 614 2024-09-17T23:59:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:59:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:59:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:59:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:21.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:21.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:59:21.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:21.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:21.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:59:21.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:59:22.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 615 2024-09-17T23:59:22.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:22.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:22.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:59:22.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:22.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:22.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:59:22.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:59:22.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:22.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:59:22.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:22.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:22.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:59:22.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:59:23.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 617 2024-09-17T23:59:23.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 616 2024-09-17T23:59:23.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:23.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:59:23.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:23.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:23.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:59:23.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:59:23.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:59:23.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:23.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:23.110 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-17T23:59:23.109+0000 7fac32fac640 -1 reset not still connected to 0x55a85217d450 2024-09-17T23:59:24.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:24.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:59:24.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:24.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:24.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:59:24.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:59:24.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 618 2024-09-17T23:59:24.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:24.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:59:24.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:24.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:24.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:59:24.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:59:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 619 2024-09-17T23:59:24.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:24.121 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-17T23:59:24.121+0000 7f849c00d640 -1 reset not still connected to 0x5629ae8ee410 2024-09-17T23:59:24.121 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-17T23:59:24.121+0000 7f849c00d640 -1 reset not still connected to 0x5629b146ed00 2024-09-17T23:59:24.121 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-17T23:59:24.121+0000 7f849c00d640 -1 reset not still connected to 0x5629c733bc70 2024-09-17T23:59:25.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:25.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-17T23:59:25.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:25.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:25.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-17T23:59:25.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-17T23:59:25.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 621 2024-09-17T23:59:25.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 620 2024-09-17T23:59:25.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 622 2024-09-17T23:59:25.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:25.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:25.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:26.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 624 2024-09-17T23:59:26.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 623 2024-09-17T23:59:26.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:26.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:59:26.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:26.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:26.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:59:26.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:59:26.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:26.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:59:26.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:26.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:26.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:59:26.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:59:26.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:26.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:27.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 625 2024-09-17T23:59:27.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-17T23:59:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-17T23:59:27.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-17T23:59:27.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:27.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:59:27.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:27.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:27.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:59:27.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:59:28.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 626 2024-09-17T23:59:28.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 627 2024-09-17T23:59:28.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:28.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-17T23:59:28.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:28.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:28.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-17T23:59:28.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-17T23:59:28.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:28.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:59:28.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:28.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:28.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:59:28.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:59:28.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:28.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:28.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:59:28.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:28.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:28.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:59:28.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:59:28.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:29.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 629 2024-09-17T23:59:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:59:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:29.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:59:29.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 628 2024-09-17T23:59:29.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:29.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-17T23:59:29.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:29.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:29.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-17T23:59:29.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-17T23:59:29.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:29.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:30.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:59:30.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 41452 2024-09-17T23:59:30.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 631 2024-09-17T23:59:30.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 630 2024-09-17T23:59:30.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 632 2024-09-17T23:59:30.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:30.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:30.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:31.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:31.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-17T23:59:31.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:31.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:31.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-17T23:59:31.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-17T23:59:31.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 633 2024-09-17T23:59:31.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 634 2024-09-17T23:59:31.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:31.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:59:31.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:31.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:31.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:59:31.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:59:31.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:31.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:32.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:32.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-17T23:59:32.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:32.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:32.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-17T23:59:32.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-17T23:59:32.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:59:32.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 41586 2024-09-17T23:59:33.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:33.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-17T23:59:33.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:33.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:33.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-17T23:59:33.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-17T23:59:33.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 635 2024-09-17T23:59:33.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 636 2024-09-17T23:59:33.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:33.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:59:33.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:33.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:33.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:59:33.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:59:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:59:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 41739 2024-09-17T23:59:33.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:33.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:34.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 637 2024-09-17T23:59:34.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-17T23:59:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-17T23:59:34.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-17T23:59:34.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:59:34.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 41872 2024-09-17T23:59:34.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:35.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 638 2024-09-17T23:59:35.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 639 2024-09-17T23:59:35.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:35.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 640 2024-09-17T23:59:36.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:36.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:36.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-17T23:59:36.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:36.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:36.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-17T23:59:36.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-17T23:59:36.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:59:36.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 42046 2024-09-17T23:59:37.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:59:37.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 42200 2024-09-17T23:59:38.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 641 2024-09-17T23:59:38.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:38.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:59:38.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 42354 2024-09-17T23:59:38.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-17T23:59:38.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-17T23:59:38.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-17T23:59:38.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-17T23:59:38.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-17T23:59:38.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-17T23:59:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:59:39.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 42508 2024-09-17T23:59:40.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 642 2024-09-17T23:59:40.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-17T23:59:41.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-17T23:59:41.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 42740 2024-09-17T23:59:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-17T23:59:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-17T23:59:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 41320 2024-09-18T00:02:02.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:02:02.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:02:02.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:02:02.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 41453 2024-09-18T00:02:03.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:02:03.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:02:03.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:02:03.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 41587 2024-09-18T00:02:04.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:02:04.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:02:04.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:02:04.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 41740 2024-09-18T00:02:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:02:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:02:05.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:02:05.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 41873 2024-09-18T00:02:07.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:02:07.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:02:07.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:02:07.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 42047 2024-09-18T00:02:08.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:02:08.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:02:08.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:02:08.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 42201 2024-09-18T00:02:09.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:02:09.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:02:09.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:02:09.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 42355 2024-09-18T00:02:10.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:02:10.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:02:10.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:02:10.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 42509 2024-09-18T00:02:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:02:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:02:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:02:12.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 42741 2024-09-18T00:02:14.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:02:14.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:02:14.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:02:14.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:02:14.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:02:14.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:02:14.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 1 2024-09-18T00:02:14.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd0 2024-09-18T00:02:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:02:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary1 2024-09-18T00:02:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary1 2024-09-18T00:02:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:02:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary1 2024-09-18T00:02:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:02:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 2 2024-09-18T00:02:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd1 2024-09-18T00:02:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary2 2024-09-18T00:02:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary2 2024-09-18T00:02:14.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:02:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 3 2024-09-18T00:02:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:02:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary2 2024-09-18T00:02:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:02:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd2 2024-09-18T00:02:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary3 2024-09-18T00:02:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary3 2024-09-18T00:02:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:02:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary3 2024-09-18T00:02:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:02:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:02:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 4 2024-09-18T00:02:14.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd3 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary4 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary4 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary4 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 5 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 6 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd4 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary5 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:02:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary5 2024-09-18T00:02:14.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:02:14.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary5 2024-09-18T00:02:14.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd5 2024-09-18T00:02:14.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary6 2024-09-18T00:02:14.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary6 2024-09-18T00:02:14.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:02:14.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary6 2024-09-18T00:02:14.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:02:14.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:02:14.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 7 2024-09-18T00:02:14.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:02:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:02:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd6 2024-09-18T00:02:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary7 2024-09-18T00:02:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary7 2024-09-18T00:02:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:02:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:02:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:02:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary7 2024-09-18T00:02:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:02:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:02:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 48748 2024-09-18T00:02:14.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 10 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd9 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary10 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary10 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary10 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 9 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd8 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary9 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary9 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary9 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 8 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd7 2024-09-18T00:02:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary8 2024-09-18T00:02:14.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary8 2024-09-18T00:02:14.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:02:14.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary8 2024-09-18T00:02:14.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:02:14.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:02:14.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:02:14.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:02:14.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:02:14.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:02:14.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:02:14.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:02:14.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:02:14.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:02:14.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:02:14.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:02:14.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:02:14.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:02:14.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:02:14.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:02:14.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:02:14.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:02:14.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:02:14.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:03:09.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=1f2b57cfb68293ce69a98b0f1b27edbf 2024-09-18T00:03:09.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary5 2024-09-18T00:03:11.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary5 2024-09-18T00:03:11.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:03:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary5 2024-09-18T00:03:13.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=fc27f1be8349d541453b060cc6115209 2024-09-18T00:03:13.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary4 2024-09-18T00:03:13.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:03:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+unknown 2024-09-18T00:03:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:03:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:13.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:14.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:03:14.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:14.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:14.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:03:14.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:14.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:14.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:14.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:14.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:14.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:03:14.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary4 2024-09-18T00:03:14.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary4 2024-09-18T00:03:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:03:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9d12657b-7828-4919-923a-59e2a9847f2c 2024-09-18T00:03:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:14 2024-09-18T00:03:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:14' 2024-09-18T00:03:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:15.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=5ffcb2fb469e533ad0aaa06653448f65 2024-09-18T00:03:15.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary3 2024-09-18T00:03:15.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:15.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:15.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:15.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:15.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:15.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:15.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:15.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:15.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:15.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:15.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:15.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:14' 2024-09-18T00:03:15.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+unknown '2024-09-18 00:03:14' 2024-09-18T00:03:15.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:15.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:15.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:03:15.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:15.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:14' 2024-09-18T00:03:15.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:15.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:15.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:15.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=5d7a6df371edfcf1243e3c7036a2a943 2024-09-18T00:03:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary9 2024-09-18T00:03:15.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:14cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":1728512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617566,"remote_snapshot_timestamp":1726617566,"replay_state":"idle"}2024-09-18 00:03:09image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:14cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2117632.0,"last_snapshot_bytes":2576384,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617579,"remote_snapshot_timestamp":1726617579,"replay_state":"idle"}2024-09-18 00:03:09image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:14cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}2024-09-18 00:03:09image-primary3485ee3af-7702-462b-9378-2e2be259b434up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:14cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":966656.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617569,"remote_snapshot_timestamp":1726617569,"replay_state":"idle"}2024-09-18 00:03:09image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:14cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2140160.0,"last_snapshot_bytes":2580480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617570,"remote_snapshot_timestamp":1726617570,"replay_state":"idle"}2024-09-18 00:03:09image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:14cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:14image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:14cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3409920.0,"last_snapshot_bytes":5021696,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617573,"remote_snapshot_timestamp":1726617573,"replay_state":"idle"}2024-09-18 00:03:09image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:14cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":1757184,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617574,"remote_snapshot_timestamp":1726617574,"replay_state":"idle"}2024-09-18 00:03:09image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:14cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":962560.0,"last_snapshot_bytes":1724416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617575,"remote_snapshot_timestamp":1726617575,"replay_state":"idle"}2024-09-18 00:03:09image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:14cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2144256.0,"last_snapshot_bytes":2613248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617576,"remote_snapshot_timestamp":1726617576,"replay_state":"idle"}2024-09-18 00:03:09' 2024-09-18T00:03:15.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:15.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:03:15.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:14' 2024-09-18T00:03:15.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:03:15.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:15.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:15.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:15.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:15.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:15.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:15.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary5 up+unknown 2024-09-18T00:03:15.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:15.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:15.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:03:15.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:15.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:15.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:15.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:15.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:15.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=a4fd9330e0b2cc18a8b8fddf6b9168b1 2024-09-18T00:03:15.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary10 2024-09-18T00:03:16.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:03:16.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+unknown 2024-09-18T00:03:16.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:16.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:16.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:03:16.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:16.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:16.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:16.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:16.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:16.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary9 2024-09-18T00:03:16.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:03:16.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary9 2024-09-18T00:03:16.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary5 up+unknown '' '' 2024-09-18T00:03:16.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:16.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:16.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:03:16.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:16.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:16.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:16.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:16.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:16.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary5 2024-09-18T00:03:16.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=9f0f98f3287cc40eadde296fb3836cd6 2024-09-18T00:03:16.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary6 2024-09-18T00:03:16.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=6b147a4a2469c71c9a8cf99f4a6dad88 2024-09-18T00:03:16.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary8 2024-09-18T00:03:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary3 2024-09-18T00:03:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:03:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary3 2024-09-18T00:03:16.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:03:16.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9d12657b-7828-4919-923a-59e2a9847f2c 2024-09-18T00:03:16.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:16.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:16.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:16.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:14 2024-09-18T00:03:16.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:16.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:03:16.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:16.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:16.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:14' 2024-09-18T00:03:16.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:16.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:16.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:16.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:16.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:16.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:16.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:16.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:16.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:16.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:16.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:16.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:14' 2024-09-18T00:03:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary5 up+unknown '2024-09-18 00:03:14' 2024-09-18T00:03:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:03:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:14' 2024-09-18T00:03:16.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:16.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:16.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:17.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary10 2024-09-18T00:03:17.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:17.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:17.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:03:17.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary10 2024-09-18T00:03:17.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING2842614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":1728512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617566,"remote_snapshot_timestamp":1726617566,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:14image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2117632.0,"last_snapshot_bytes":2576384,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617579,"remote_snapshot_timestamp":1726617579,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:14image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:14image-primary3485ee3af-7702-462b-9378-2e2be259b434up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":966656.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617569,"remote_snapshot_timestamp":1726617569,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:14image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:17cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:14image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:14image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3409920.0,"last_snapshot_bytes":5021696,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617573,"remote_snapshot_timestamp":1726617573,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:14image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":1757184,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617574,"remote_snapshot_timestamp":1726617574,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:14image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":962560.0,"last_snapshot_bytes":1724416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617575,"remote_snapshot_timestamp":1726617575,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:14image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2144256.0,"last_snapshot_bytes":2613248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617576,"remote_snapshot_timestamp":1726617576,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:14' 2024-09-18T00:03:17.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:17.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:03:17.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:14' 2024-09-18T00:03:17.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:03:17.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:17.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:17.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:17.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:17.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:17.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:17.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary5 2024-09-18T00:03:17.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:17.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:17.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:03:17.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:03:17.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary5 2024-09-18T00:03:17.478 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:03:17.479+0000 7f849c00d640 -1 reset not still connected to 0x5629c733bc70 2024-09-18T00:03:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:03:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:03:17.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:17.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:17.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:17.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:17.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:17.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:03:17.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=2801b2847a0c18795e33176268b971bb 2024-09-18T00:03:17.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary1 2024-09-18T00:03:17.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=83ccb016dcef36fc62b3963216fc0fcb 2024-09-18T00:03:17.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary7 2024-09-18T00:03:17.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary6 2024-09-18T00:03:17.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:17.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:17.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:03:17.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary6 2024-09-18T00:03:17.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary8 2024-09-18T00:03:17.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:17.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:17.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:03:17.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary8 2024-09-18T00:03:17.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:03:17.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 3a58ab54-d559-485b-ad84-79ecd4d5dae7 2024-09-18T00:03:17.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:03:17.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:03:17.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:17.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:14 2024-09-18T00:03:17.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:17.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:17.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:17.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:03:17.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:17' 2024-09-18T00:03:17.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:17.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:03:17.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:17.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:03:17.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:03:17.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:03:17.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:03:18.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd2 2024-09-18T00:03:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary1 2024-09-18T00:03:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:03:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary1 2024-09-18T00:03:18.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:03:18.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:03:18.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary7 2024-09-18T00:03:18.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:18.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:18.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:03:18.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary7 2024-09-18T00:03:18.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:03:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:03:18.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:03:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+unknown 2024-09-18T00:03:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:03:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:18.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:18.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+unknown 2024-09-18T00:03:18.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:18.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:18.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:03:18.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:18.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:18.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:18.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:18.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:18.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+unknown 2024-09-18T00:03:18.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:18.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:18.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:03:18.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:18.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:18.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:18.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:18.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:19.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:03:19.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:03:19.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+unknown 2024-09-18T00:03:19.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:19.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:19.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:03:19.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:19.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:19.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:19.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:19.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T00:03:19.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:19.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:19.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:03:19.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:19.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:19.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:19.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:19.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T00:03:19.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:19.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:19.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:03:19.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:19.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:19.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:19.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:19.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:19.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:03:19.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+unknown 2024-09-18T00:03:19.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:19.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:19.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:03:19.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:19.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:19.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:03:19.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T00:03:19.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:19.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:19.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:03:19.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:19.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:19.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:19.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:19.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:19.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:03:19.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=12483a6378d306a272e38bdc072554e3 2024-09-18T00:03:19.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary2 2024-09-18T00:03:19.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:03:19.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 527eda38-0231-4aa6-80fd-b8db0934cf7b 2024-09-18T00:03:19.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:19.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:19.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:19.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:19.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:19.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:19.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:19.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:19.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19' 2024-09-18T00:03:19.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:19.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:19.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:19.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:19.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:19.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:19.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:19.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:19.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:19.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:19.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:19.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:03:19.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 91a54e9f-7ef4-41db-964a-c84ca41a65ce 2024-09-18T00:03:19.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:19.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:19.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:19.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:19.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:19.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:19.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:19.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:19.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19' 2024-09-18T00:03:19.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:19.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:19.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:19.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:19.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:19.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:19.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:19.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:19.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+unknown '2024-09-18 00:03:19' 2024-09-18T00:03:19.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:19.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:19.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:03:19.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:19.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:19' 2024-09-18T00:03:19.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:19.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:19.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:19.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:19.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:19.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:19.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:19.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:19.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:19.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:19.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+unknown '2024-09-18 00:03:19' 2024-09-18T00:03:19.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:19.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:19.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:03:19.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:19.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:19' 2024-09-18T00:03:19.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:19.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:19.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:19.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:03:19.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 485ee3af-7702-462b-9378-2e2be259b434 2024-09-18T00:03:19.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:19.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:19.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:19.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:19.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:19.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:19.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:19.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:19.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19' 2024-09-18T00:03:19.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:19.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:19.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:19.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:19.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:19.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:19.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:19.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:19.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:19.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:19.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+unknown '2024-09-18 00:03:19' 2024-09-18T00:03:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:19.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:03:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:19' 2024-09-18T00:03:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:03:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:03:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:19.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:03:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":1728512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617566,"remote_snapshot_timestamp":1726617566,"replay_state":"idle"}2024-09-18 00:03:09image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}2024-09-18 00:03:09image-primary3485ee3af-7702-462b-9378-2e2be259b434up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:03:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":1757184,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617574,"remote_snapshot_timestamp":1726617574,"replay_state":"idle"}2024-09-18 00:03:09image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:03:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19' 2024-09-18T00:03:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:19.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:03:19.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:19.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:03:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:19.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:19.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:19.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary10 up+unknown 2024-09-18T00:03:19.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:19.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:19.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:03:19.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:19.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:19.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:19.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:19.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:19.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:03:19.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 3a58ab54-d559-485b-ad84-79ecd4d5dae7 2024-09-18T00:03:19.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:19.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:19.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:19.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19' 2024-09-18T00:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:19.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:19.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":1728512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617566,"remote_snapshot_timestamp":1726617566,"replay_state":"idle"}2024-09-18 00:03:09image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}2024-09-18 00:03:09image-primary3485ee3af-7702-462b-9378-2e2be259b434up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+un+ '[' -n '' ']' 2024-09-18T00:03:19.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:knownremote image is not primary2024-09-18 00:03:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:03:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":1757184,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617574,"remote_snapshot_timestamp":1726617574,"replay_state":"idle"}2024-09-18 00:03:09image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:03:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19' 2024-09-18T00:03:19.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:19.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:19.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:03:19.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:19.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:03:19.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:19.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:19.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary9 up+unknown 2024-09-18T00:03:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:03:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:19.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:19.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:19.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:20.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:20.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:20.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:20.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:20.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+unknown '2024-09-18 00:03:19' 2024-09-18T00:03:20.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:20.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:03:20.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:19' 2024-09-18T00:03:20.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:20.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:20.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:20.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":1728512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617566,"remote_snapshot_timestamp":1726617566,"replay_state":"idle"}2024-09-18 00:03:09image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}2024-09-18 00:03:09image-primary3485ee3af-7702-462b-9378-2e2be259b434up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:03:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":1757184,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617574,"remote_snapshot_timestamp":1726617574,"replay_state":"idle"}2024-09-18 00:03:09image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:03:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19' 2024-09-18T00:03:20.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:20.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:03:20.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:20.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:03:20.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:20.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:20.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:20.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary3 up+unknown 2024-09-18T00:03:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:03:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:20.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":1728512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617566,"remote_snapshot_timestamp":1726617566,"replay_state":"idle"}2024-09-18 00:03:09image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}2024-09-18 00:03:09image-primary3485ee3af-7702-462b-9378-2e2be259b434up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:03:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":1757184,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617574,"remote_snapshot_timestamp":1726617574,"replay_state":"idle"}2024-09-18 00:03:09image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:03:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19' 2024-09-18T00:03:20.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:20.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:03:20.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:20.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:03:20.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:20.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:20.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:20.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:20.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:20.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:20.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary4 up+unknown 2024-09-18T00:03:20.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:20.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:03:20.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:20.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:20.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:20.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:20.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:03:20.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+unknown 2024-09-18T00:03:20.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:20.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:03:20.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:20.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:20.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:20.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:20.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:03:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+unknown 2024-09-18T00:03:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:03:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:03:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:03:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:03:20.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T00:03:20.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:20.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:03:20.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:20.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:20.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:20.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:20.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:03:20.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:03:20.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 11e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7e 2024-09-18T00:03:20.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:20.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:20.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:20.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:20.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:20.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:20.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:20.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:03:20.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19' 2024-09-18T00:03:20.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:20.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:20.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:20.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:20.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:20.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+unknown '2024-09-18 00:03:19' 2024-09-18T00:03:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:03:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:19' 2024-09-18T00:03:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:20.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:20.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:03:20.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 631249db-2f37-4574-b5c1-3ddb41b94859 2024-09-18T00:03:20.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:20.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:20.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:20.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:20.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:20.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:20.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:20.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:03:20.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19' 2024-09-18T00:03:20.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:20.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:20.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:20.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:20.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:20.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:20.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:20.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:20.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:20.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:20.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:20.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+unknown '2024-09-18 00:03:19' 2024-09-18T00:03:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:03:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:19' 2024-09-18T00:03:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:20.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary2 2024-09-18T00:03:20.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:20.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:03:20.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary2 2024-09-18T00:03:20.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":1728512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617566,"remote_snapshot_timestamp":1726617566,"replay_state":"idle"}2024-09-18 00:03:09image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}2024-09-18 00:03:09image-primary3485ee3af-7702-462b-9378-2e2be259b434up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:03:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":1757184,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617574,"remote_snapshot_timestamp":1726617574,"replay_state":"idle"}2024-09-18 00:03:09image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:03:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19' 2024-09-18T00:03:20.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:20.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:03:20.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":1728512,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617566,"remote_snapshot_timestamp":1726617566,"replay_state":"idle"}2024-09-18 00:03:09image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}2024-09-18 00:03:09image-primary3485ee3af-7702-462b-9378-2e2be259b434up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:03:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":976896.0,"last_snapshot_bytes":1757184,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617574,"remote_snapshot_timestamp":1726617574,"replay_state":"idle"}2024-09-18 00:03:09image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:03:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:19' 2024-09-18T00:03:20.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:20.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:03:20.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:03:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:03:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:20.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:20.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:20.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:20.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:20.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary6 up+unknown 2024-09-18T00:03:20.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:20.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary8 up+unknown 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:20.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:20.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:20.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:20.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary10 up+unknown '' '' 2024-09-18T00:03:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:03:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:20.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary10 2024-09-18T00:03:20.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary9 up+unknown '' '' 2024-09-18T00:03:20.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:20.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:20.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:03:20.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:20.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:20.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:20.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:20.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:20.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary9 2024-09-18T00:03:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:03:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 527eda38-0231-4aa6-80fd-b8db0934cf7b 2024-09-18T00:03:21.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:21.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:21.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:21.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:21.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:03:21.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:21.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19' 2024-09-18T00:03:21.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:21.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:21.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary10 up+unknown '2024-09-18 00:03:19' 2024-09-18T00:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:03:21.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:21.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:19' 2024-09-18T00:03:21.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:21.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:21.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:03:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 91a54e9f-7ef4-41db-964a-c84ca41a65ce 2024-09-18T00:03:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:03:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19' 2024-09-18T00:03:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:21.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:21.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:21.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:21.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:21.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:21.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:21.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:21.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:21.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary9 up+unknown '2024-09-18 00:03:19' 2024-09-18T00:03:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:03:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:19' 2024-09-18T00:03:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:21.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary3 up+unknown '' '' 2024-09-18T00:03:21.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:03:21.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:21.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:21.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:21.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:21.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:21.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary3 2024-09-18T00:03:21.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:20cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary3485ee3af-7702-462b-9378-2e2be259b434up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:20cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19' 2024-09-18T00:03:21.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:21.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:03:21.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:21.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:03:21.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:03:21.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 485ee3af-7702-462b-9378-2e2be259b434 2024-09-18T00:03:21.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:21.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:21.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:21.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:21.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:03:21.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:21.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19' 2024-09-18T00:03:21.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:21.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:21.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:21.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:21.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:21.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:21.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:21.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:21.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:21.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:21.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:21.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:21.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:21.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary10 2024-09-18T00:03:21.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:03:21.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:03:21.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary10 2024-09-18T00:03:21.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:21.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:20cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary3485ee3af-7702-462b-9378-2e2be259b434up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:20cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19' 2024-09-18T00:03:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:21.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:21.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:03:21.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:21.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary3 up+unknown '2024-09-18 00:03:19' 2024-09-18T00:03:21.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:03:21.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:21.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:19' 2024-09-18T00:03:21.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:21.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:03:21.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:21.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:21.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:21.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:21.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:21.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:21.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:21.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:21.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary9 2024-09-18T00:03:21.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:03:21.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:03:21.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary9 2024-09-18T00:03:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary4 up+unknown '' '' 2024-09-18T00:03:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:03:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary4 2024-09-18T00:03:21.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:03:21.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 3a58ab54-d559-485b-ad84-79ecd4d5dae7 2024-09-18T00:03:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:03:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19' 2024-09-18T00:03:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:21.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:21.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:21.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:21.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:21.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:21.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:21.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:21.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:21.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:21.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:21.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:21.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:21.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary4 up+unknown '2024-09-18 00:03:19' 2024-09-18T00:03:21.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:03:21.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:21.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:19' 2024-09-18T00:03:21.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:21.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:21.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:20cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary3485ee3af-7702-462b-9378-2e2be259b434up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:20cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19' 2024-09-18T00:03:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:21.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:03:21.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:21.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:03:21.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:21.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:21.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:21.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary3 2024-09-18T00:03:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:03:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:03:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary3 2024-09-18T00:03:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:20cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary3485ee3af-7702-462b-9378-2e2be259b434up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:20cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19' 2024-09-18T00:03:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:21.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:03:21.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:21.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:03:21.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:21.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:21.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:21.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:21.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:21.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:21.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary4 2024-09-18T00:03:21.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:03:21.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:03:21.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary4 2024-09-18T00:03:21.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:03:21.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:21.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:03:21.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:21.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:21.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:21.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:21.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:21.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:03:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:03:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:03:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:21.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:21.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:21.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:21.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:21.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:03:21.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:03:21.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 40e08d00-c76c-475b-a9ba-1e4c6459df5a 2024-09-18T00:03:21.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:03:21.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:03:21.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:21.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:21.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:21.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:21.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:03:21.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:20' 2024-09-18T00:03:21.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:21.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:03:21.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:21.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:03:21.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:03:21.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 86e615f2-9a52-4cc1-a03c-46ae2effd0d8 2024-09-18T00:03:21.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:03:21.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:03:21.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:21.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:21.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:21.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:21.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:03:21.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:20' 2024-09-18T00:03:21.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:21.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:03:21.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:21.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:03:21.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary6 up+unknown '' '' 2024-09-18T00:03:21.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary8 up+unknown '' '' 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:21.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary6 2024-09-18T00:03:21.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:21.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary8 2024-09-18T00:03:21.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:03:21.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:03:21.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:03:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:03:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 631249db-2f37-4574-b5c1-3ddb41b94859 2024-09-18T00:03:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:03:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:03:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:21.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19' 2024-09-18T00:03:21.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:21.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:21.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:21.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:03:21.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 11e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7e 2024-09-18T00:03:21.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:03:21.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:21.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:21.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:21.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:03:21.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:21.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:21.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19' 2024-09-18T00:03:21.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:21.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:03:21.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:21.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:21.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:21.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:21.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:21.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:21.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:21.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:03:21.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:21.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:21.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:21.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:21.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:21.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:03:21.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:21.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:21.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary6 up+unknown '2024-09-18 00:03:19' 2024-09-18T00:03:21.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:03:21.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:21.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:19' 2024-09-18T00:03:21.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:21.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:21.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:21.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:21.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:21.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary8 up+unknown '2024-09-18 00:03:19' 2024-09-18T00:03:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:03:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:19' 2024-09-18T00:03:21.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:21.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:03:21.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:03:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:22.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:03:22.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:03:22.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:03:22.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:03:22.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:03:22.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:03:22.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:03:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+unknown 2024-09-18T00:03:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:03:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:22.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:20cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary3485ee3af-7702-462b-9378-2e2be259b434up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:20cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19' 2024-09-18T00:03:22.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:22.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:03:22.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd0 2024-09-18T00:03:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:03:22.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:03:22.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:22.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:03:22.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:22.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:22.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:20cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1697792.0,"last_snapshot_bytes":1667072,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617568,"remote_snapshot_timestamp":1726617568,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:03:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary3485ee3af-7702-462b-9378-2e2be259b434up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:20cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:03:19image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:19cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:19' 2024-09-18T00:03:22.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:22.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:03:22.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:22.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:22.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:22.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:22.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary6 2024-09-18T00:03:22.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:22.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:22.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:03:22.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:03:22.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary6 2024-09-18T00:03:22.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:19' 2024-09-18T00:03:22.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:03:22.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:22.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:22.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:22.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:22.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary8 2024-09-18T00:03:22.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:22.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:22.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:03:22.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:03:22.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary8 2024-09-18T00:03:22.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd1 2024-09-18T00:03:22.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:03:22.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:03:22.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd3 2024-09-18T00:03:22.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:03:22.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:03:22.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd4 2024-09-18T00:03:22.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:03:22.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:03:22.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:03:22.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:03:23.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:03:23.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:03:23.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:03:23.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:03:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:03:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:03:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:03:23.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:03:23.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: be7259a5-f0ba-4f85-b82a-ee2c6fa0005e 2024-09-18T00:03:23.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:03:23.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:03:23.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:23.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:23.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:23.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:23.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:23.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:03:23.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:22' 2024-09-18T00:03:23.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:23.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:03:23.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:23.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:03:23.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd5 2024-09-18T00:03:23.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:03:23.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:03:23.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd6 2024-09-18T00:03:23.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:03:23.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:03:23.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:03:23.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:23.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:23.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:03:23.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:23.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:23.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:23.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:03:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:03:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:03:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:23.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:23.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:03:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:03:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 40e08d00-c76c-475b-a9ba-1e4c6459df5a 2024-09-18T00:03:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:03:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:03:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:23.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:23.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:03:23.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:20' 2024-09-18T00:03:23.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:23.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:03:23.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:23.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:03:23.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:03:23.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 86e615f2-9a52-4cc1-a03c-46ae2effd0d8 2024-09-18T00:03:23.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:03:23.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:03:23.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:23.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:19 2024-09-18T00:03:23.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:23.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:23.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:23.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:03:23.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:20' 2024-09-18T00:03:23.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:23.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:03:23.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:23.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:25.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:03:25.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:03:25.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: be7259a5-f0ba-4f85-b82a-ee2c6fa0005e 2024-09-18T00:03:25.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:25.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:24 2024-09-18T00:03:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:24' 2024-09-18T00:03:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:25.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:25.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:25.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:25.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:25.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:25.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:25.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:25.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:25.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:25.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:24' 2024-09-18T00:03:25.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+unknown '2024-09-18 00:03:24' 2024-09-18T00:03:25.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:25.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:25.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:03:25.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:25.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:24' 2024-09-18T00:03:25.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:25.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:25.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:26.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:24image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:24image-primary3485ee3af-7702-462b-9378-2e2be259b434up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:24image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24' 2024-09-18T00:03:26.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:26.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:03:26.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:24' 2024-09-18T00:03:26.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:03:26.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:26.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:26.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:26.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:26.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:26.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:26.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary2 up+unknown 2024-09-18T00:03:26.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:26.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:26.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:03:26.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:26.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:26.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:26.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:26.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:27.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary2 up+unknown '' '' 2024-09-18T00:03:27.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:27.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:27.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:03:27.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:27.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:27.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:27.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:27.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:27.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary2 2024-09-18T00:03:27.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:03:27.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: be7259a5-f0ba-4f85-b82a-ee2c6fa0005e 2024-09-18T00:03:27.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:27.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:27.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:27.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:24 2024-09-18T00:03:27.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:27.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:03:27.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:27.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:27.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:24' 2024-09-18T00:03:27.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:27.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:27.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:27.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:27.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:27.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:27.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:27.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:27.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:24' 2024-09-18T00:03:27.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary2 up+unknown '2024-09-18 00:03:24' 2024-09-18T00:03:27.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:03:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:24' 2024-09-18T00:03:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:27.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:27.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:24cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:24image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:24cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}2024-09-18 00:03:24image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:24cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:24image-primary3485ee3af-7702-462b-9378-2e2be259b434up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:24cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}2024-09-18 00:03:24image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:24cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}2024-09-18 00:03:24image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:24cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:24cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}2024-09-18 00:03:24image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:24cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:24image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:24cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}2024-09-18 00:03:24image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:24cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}2024-09-18 00:03:24' 2024-09-18T00:03:27.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:27.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:03:27.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:24' 2024-09-18T00:03:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:03:27.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:27.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:27.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:27.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:27.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:27.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:27.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary2 2024-09-18T00:03:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:03:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:03:27.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary2 2024-09-18T00:03:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:03:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:27.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:03:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:27.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:03:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:03:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:03:27.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:27.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:03:28.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:03:28.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 40e08d00-c76c-475b-a9ba-1e4c6459df5a 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:24 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:24' 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 86e615f2-9a52-4cc1-a03c-46ae2effd0d8 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:24 2024-09-18T00:03:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:03:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:24' 2024-09-18T00:03:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:28.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:28.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:28.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:28.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:28.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:28.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:28.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:28.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:28.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:28.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:28.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:28.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:28.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:28.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:28.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:24' 2024-09-18T00:03:28.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+unknown '2024-09-18 00:03:24' 2024-09-18T00:03:28.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:28.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:28.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:03:28.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:28.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:24' 2024-09-18T00:03:28.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:28.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:28.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:28.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:28.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:28.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:24' 2024-09-18T00:03:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+unknown '2024-09-18 00:03:24' 2024-09-18T00:03:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:03:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:03:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:24' 2024-09-18T00:03:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:28.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:28.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:24image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:24image-primary3485ee3af-7702-462b-9378-2e2be259b434up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:24image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24' 2024-09-18T00:03:28.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:24image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:24image-primary3485ee3af-7702-462b-9378-2e2be259b434up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:19cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:03:24image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:03:24cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:03:24' 2024-09-18T00:03:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:03:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:03:28.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:24' 2024-09-18T00:03:28.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:24' 2024-09-18T00:03:28.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:03:28.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:03:28.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:28.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:28.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:28.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:28.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:28.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary1 up+unknown 2024-09-18T00:03:28.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:28.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:28.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:03:28.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:28.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:28.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:28.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:28.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:28.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:28.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:28.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:28.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:28.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:28.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:28.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary7 up+unknown 2024-09-18T00:03:28.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:28.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:28.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:03:28.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:28.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:28.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:28.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:03:28.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:03:28.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:03:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:03:28.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:03:29.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd7 2024-09-18T00:03:29.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:03:29.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:03:29.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary1 up+unknown '' '' 2024-09-18T00:03:29.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:29.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:29.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:03:29.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:29.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:29.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:29.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:29.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:29.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary1 2024-09-18T00:03:29.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary7 up+unknown '' '' 2024-09-18T00:03:29.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:29.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:29.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:03:29.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:29.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:03:29.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:03:29.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:29.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:29.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary7 2024-09-18T00:03:29.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:03:29.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 40e08d00-c76c-475b-a9ba-1e4c6459df5a 2024-09-18T00:03:29.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:29.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:29.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:29.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:29 2024-09-18T00:03:29.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:29.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:03:29.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:29.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:29.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:29' 2024-09-18T00:03:29.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:29.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:29.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:03:29.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 86e615f2-9a52-4cc1-a03c-46ae2effd0d8 2024-09-18T00:03:29.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:29.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:29.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:03:29.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:29 2024-09-18T00:03:29.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:03:29.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:03:29.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:03:29.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:03:29.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:03:29' 2024-09-18T00:03:29.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:03:29.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:29.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:29.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:29.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:03:29.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:03:29.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:03:29.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:03:29.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:29.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:29.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:29.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:03:29.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:29.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:03:29.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:29.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:29.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:29.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:03:29.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:03:29.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:29' 2024-09-18T00:03:29.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary1 up+unknown '2024-09-18 00:03:29' 2024-09-18T00:03:29.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:29.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:29.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:03:29.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:29.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:29' 2024-09-18T00:03:29.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:29.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:03:29.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:29.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:29' 2024-09-18T00:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary7 up+unknown '2024-09-18 00:03:29' 2024-09-18T00:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:03:29' 2024-09-18T00:03:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:03:29.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:03:29.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:03:29.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING2842614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:29image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}2024-09-18 00:03:24image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617808,"replay_state":"idle"}2024-09-18 00:03:29image-primary3485ee3af-7702-462b-9378-2e2be259b434up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}2024-09-18 00:03:24image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}2024-09-18 00:03:24image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}2024-09-18 00:03:24image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:29image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}2024-09-18 00:03:24image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}2024-09-18 00:03:24' 2024-09-18T00:03:29.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:29.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:03:29.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:29' 2024-09-18T00:03:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:03:29.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING2842614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:29image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}2024-09-18 00:03:24image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617808,"replay_state":"idle"}2024-09-18 00:03:29image-primary3485ee3af-7702-462b-9378-2e2be259b434up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}2024-09-18 00:03:24image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}2024-09-18 00:03:24image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}2024-09-18 00:03:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}2024-09-18 00:03:24image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:03:29image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}2024-09-18 00:03:24image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:03:29cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}2024-09-18 00:03:24' 2024-09-18T00:03:29.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:03:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:29.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:30.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:03:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary1 2024-09-18T00:03:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:30.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:30.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:03:30.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:03:30.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary1 2024-09-18T00:03:30.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:03:29' 2024-09-18T00:03:30.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:03:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:03:30.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:03:30.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:03:30.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:03:30.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:30.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:03:30.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary7 2024-09-18T00:03:30.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:03:30.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:03:30.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:03:30.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:03:30.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary7 2024-09-18T00:03:30.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:03:30.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:03:30.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:03:30.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:03:30.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:03:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:03:31.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd8 2024-09-18T00:03:31.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:03:31.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:03:31.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd9 2024-09-18T00:03:31.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:03:31.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:03:39.142 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:03:39.146+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735aedd0 2024-09-18T00:05:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=1f2b57cfb68293ce69a98b0f1b27edbf 2024-09-18T00:05:00.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd2 2024-09-18T00:05:02.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 1f2b57cfb68293ce69a98b0f1b27edbf != \1\f\2\b\5\7\c\f\b\6\8\2\9\3\c\e\6\9\a\9\8\b\0\f\1\b\2\7\e\d\b\f ]] 2024-09-18T00:05:06.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=5d7a6df371edfcf1243e3c7036a2a943 2024-09-18T00:05:06.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd1 2024-09-18T00:05:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 5d7a6df371edfcf1243e3c7036a2a943 != \5\d\7\a\6\d\f\3\7\1\e\d\f\c\f\1\2\4\3\e\3\c\7\0\3\6\a\2\a\9\4\3 ]] 2024-09-18T00:05:09.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=fc27f1be8349d541453b060cc6115209 2024-09-18T00:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd3 2024-09-18T00:05:10.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=5ffcb2fb469e533ad0aaa06653448f65 2024-09-18T00:05:10.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd4 2024-09-18T00:05:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ fc27f1be8349d541453b060cc6115209 != \f\c\2\7\f\1\b\e\8\3\4\9\d\5\4\1\4\5\3\b\0\6\0\c\c\6\1\1\5\2\0\9 ]] 2024-09-18T00:05:11.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 5ffcb2fb469e533ad0aaa06653448f65 != \5\f\f\c\b\2\f\b\4\6\9\e\5\3\3\a\d\0\a\a\a\0\6\6\5\3\4\4\8\f\6\5 ]] 2024-09-18T00:05:11.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=6b147a4a2469c71c9a8cf99f4a6dad88 2024-09-18T00:05:11.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd6 2024-09-18T00:05:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=a4fd9330e0b2cc18a8b8fddf6b9168b1 2024-09-18T00:05:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd0 2024-09-18T00:05:11.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=9f0f98f3287cc40eadde296fb3836cd6 2024-09-18T00:05:11.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd5 2024-09-18T00:05:12.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 6b147a4a2469c71c9a8cf99f4a6dad88 != \6\b\1\4\7\a\4\a\2\4\6\9\c\7\1\c\9\a\8\c\f\9\9\f\4\a\6\d\a\d\8\8 ]] 2024-09-18T00:05:12.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ a4fd9330e0b2cc18a8b8fddf6b9168b1 != \a\4\f\d\9\3\3\0\e\0\b\2\c\c\1\8\a\8\b\8\f\d\d\f\6\b\9\1\6\8\b\1 ]] 2024-09-18T00:05:12.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 9f0f98f3287cc40eadde296fb3836cd6 != \9\f\0\f\9\8\f\3\2\8\7\c\c\4\0\e\a\d\d\e\2\9\6\f\b\3\8\3\6\c\d\6 ]] 2024-09-18T00:05:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=12483a6378d306a272e38bdc072554e3 2024-09-18T00:05:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd7 2024-09-18T00:05:13.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=83ccb016dcef36fc62b3963216fc0fcb 2024-09-18T00:05:13.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd8 2024-09-18T00:05:14.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 12483a6378d306a272e38bdc072554e3 != \1\2\4\8\3\a\6\3\7\8\d\3\0\6\a\2\7\2\e\3\8\b\d\c\0\7\2\5\5\4\e\3 ]] 2024-09-18T00:05:14.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 83ccb016dcef36fc62b3963216fc0fcb != \8\3\c\c\b\0\1\6\d\c\e\f\3\6\f\c\6\2\b\3\9\6\3\2\1\6\f\c\0\f\c\b ]] 2024-09-18T00:05:14.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=2801b2847a0c18795e33176268b971bb 2024-09-18T00:05:14.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd9 2024-09-18T00:05:15.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 2801b2847a0c18795e33176268b971bb != \2\8\0\1\b\2\8\4\7\a\0\c\1\8\7\9\5\e\3\3\1\7\6\2\6\8\b\9\7\1\b\b ]] 2024-09-18T00:05:15.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:05:15.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 48749 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 48751 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 48754 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 48755 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 48757 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 48758 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 48761 2024-09-18T00:05:15.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:05:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 48762 2024-09-18T00:05:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:05:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 48763 2024-09-18T00:05:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:05:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:05:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary1 2024-09-18T00:05:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:05:15.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+replaying 2024-09-18T00:05:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:15.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:15.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:05:16.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+replaying '' '' 2024-09-18T00:05:16.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:16.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:16.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:16.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:16.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:16.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:16.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:16.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:16.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:05:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 86e615f2-9a52-4cc1-a03c-46ae2effd0d8 2024-09-18T00:05:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:05:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72540160.0,"last_snapshot_bytes":72540160,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617810,"remote_snapshot_timestamp":1726617810,"replay_state":"idle"} 2024-09-18T00:05:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:05:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:09 2024-09-18T00:05:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:05:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:05:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:05:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:05:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:14' 2024-09-18T00:05:16.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:05:16.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:05:16.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:05:16.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72540160.0,"last_snapshot_bytes":72540160,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617810,"remote_snapshot_timestamp":1726617810,"replay_state":"idle"} 2024-09-18T00:05:16.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:05:16.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:16.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:05:16.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:16.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:05:16.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:05:16.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:05:16.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:05:16.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:16.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+replaying '2024-09-18 00:05:09' 2024-09-18T00:05:16.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:16.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:16.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:16.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:16.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:05:09' 2024-09-18T00:05:16.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:16.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:16.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:05:16.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary186e615f2-9a52-4cc1-a03c-46ae2effd0d8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72540160.0,"last_snapshot_bytes":72540160,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617810,"remote_snapshot_timestamp":1726617810,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:14image-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69890048.0,"last_snapshot_bytes":69890048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:14image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70836224.0,"last_snapshot_bytes":70836224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617808,"remote_snapshot_timestamp":1726617808,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:14image-primary3485ee3af-7702-462b-9378-2e2be259b434up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72269824.0,"last_snapshot_bytes":72269824,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:14image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70303744.0,"last_snapshot_bytes":70303744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:14image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70090752.0,"last_snapshot_bytes":70090752,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617797,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:14image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63799296.0,"last_snapshot_bytes":63799296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:14image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617810,"remote_snapshot_timestamp":1726617810,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:14image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72302592.0,"last_snapshot_bytes":72302592,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:14image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:14' 2024-09-18T00:05:16.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:05:16.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:05:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:05:16.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:05:16.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:16.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:05:16.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:16.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:16.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary1 2024-09-18T00:05:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:05:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:16.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary1 2024-09-18T00:05:16.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary1 2024-09-18T00:05:19.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:05:19.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary1 2024-09-18T00:05:19.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:19.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:19.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:19.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:19.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:05:19.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary1 2024-09-18T00:05:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary1 2024-09-18T00:05:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:05:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary2 2024-09-18T00:05:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:05:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+replaying 2024-09-18T00:05:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:19.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:19.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:19.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:19.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:05:19.826 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:05:19.825+0000 7fac32fac640 -1 reset not still connected to 0x55a83a519e10 2024-09-18T00:05:19.827 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:05:19.825+0000 7fac32fac640 -1 reset not still connected to 0x55a8454fa680 2024-09-18T00:05:19.827 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:05:19.825+0000 7fac32fac640 -1 reset not still connected to 0x55a8475f16c0 2024-09-18T00:05:19.827 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:05:19.825+0000 7fac32fac640 -1 reset not still connected to 0x55a84ffbdee0 2024-09-18T00:05:20.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+replaying '' '' 2024-09-18T00:05:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:05:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:05:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: be7259a5-f0ba-4f85-b82a-ee2c6fa0005e 2024-09-18T00:05:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:05:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70836224.0,"last_snapshot_bytes":70836224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617808,"remote_snapshot_timestamp":1726617808,"replay_state":"idle"} 2024-09-18T00:05:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:05:20.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:09 2024-09-18T00:05:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:05:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:05:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:05:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:05:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:19' 2024-09-18T00:05:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:05:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:05:20.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:05:20.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70836224.0,"last_snapshot_bytes":70836224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617808,"remote_snapshot_timestamp":1726617808,"replay_state":"idle"} 2024-09-18T00:05:20.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:05:20.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:20.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:05:20.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:20.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:05:20.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:05:20.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:05:20.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:05:20.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:20.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+replaying '2024-09-18 00:05:09' 2024-09-18T00:05:20.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:20.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:20.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:20.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:20.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:05:09' 2024-09-18T00:05:20.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:20.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:20.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:05:20.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69890048.0,"last_snapshot_bytes":69890048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary2be7259a5-f0ba-4f85-b82a-ee2c6fa0005eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70836224.0,"last_snapshot_bytes":70836224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617808,"remote_snapshot_timestamp":1726617808,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary3485ee3af-7702-462b-9378-2e2be259b434up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72269824.0,"last_snapshot_bytes":72269824,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70303744.0,"last_snapshot_bytes":70303744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70090752.0,"last_snapshot_bytes":70090752,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617797,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63799296.0,"last_snapshot_bytes":63799296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617810,"remote_snapshot_timestamp":1726617810,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72302592.0,"last_snapshot_bytes":72302592,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19' 2024-09-18T00:05:20.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:05:20.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:05:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:05:20.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:05:20.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:20.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:05:20.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:20.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:20.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:20.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary2 2024-09-18T00:05:20.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:05:20.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:20.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:20.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary2 2024-09-18T00:05:20.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary2 2024-09-18T00:05:22.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:05:22.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary2 2024-09-18T00:05:22.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:22.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:22.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:22.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:22.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:05:22.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary2 2024-09-18T00:05:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary2 2024-09-18T00:05:22.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:22.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:05:22.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary3 2024-09-18T00:05:22.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:22.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:22.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:22.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:05:22.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+replaying 2024-09-18T00:05:22.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:22.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:22.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:22.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:22.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:22.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:22.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:22.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:05:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+replaying '' '' 2024-09-18T00:05:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:23.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:23.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:05:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:05:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 485ee3af-7702-462b-9378-2e2be259b434 2024-09-18T00:05:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:05:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72269824.0,"last_snapshot_bytes":72269824,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"} 2024-09-18T00:05:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:05:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:09 2024-09-18T00:05:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:05:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:05:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:05:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:05:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:19' 2024-09-18T00:05:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:05:23.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:05:23.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:05:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72269824.0,"last_snapshot_bytes":72269824,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"} 2024-09-18T00:05:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:05:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:05:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:05:23.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:05:23.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:05:23.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:05:23.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:23.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+replaying '2024-09-18 00:05:09' 2024-09-18T00:05:23.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:23.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:23.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:23.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:23.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:05:09' 2024-09-18T00:05:23.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:23.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:23.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:05:23.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK842544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69890048.0,"last_snapshot_bytes":69890048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary3485ee3af-7702-462b-9378-2e2be259b434up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72269824.0,"last_snapshot_bytes":72269824,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70303744.0,"last_snapshot_bytes":70303744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70090752.0,"last_snapshot_bytes":70090752,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617797,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63799296.0,"last_snapshot_bytes":63799296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617810,"remote_snapshot_timestamp":1726617810,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72302592.0,"last_snapshot_bytes":72302592,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:19' 2024-09-18T00:05:23.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:05:23.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:05:23.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:23.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:05:23.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:05:23.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:23.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:05:23.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:23.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:23.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:23.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary3 2024-09-18T00:05:23.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:05:23.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:23.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:23.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary3 2024-09-18T00:05:23.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary3 2024-09-18T00:05:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:05:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary3 2024-09-18T00:05:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:24.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:24.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:24.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:24.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:05:24.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary3 2024-09-18T00:05:24.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:24.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary3 2024-09-18T00:05:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:05:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary4 2024-09-18T00:05:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:05:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+replaying 2024-09-18T00:05:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:24.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:05:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+replaying '' '' 2024-09-18T00:05:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:25.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:25.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:25.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:05:26.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:05:26.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 3a58ab54-d559-485b-ad84-79ecd4d5dae7 2024-09-18T00:05:26.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:05:26.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70303744.0,"last_snapshot_bytes":70303744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"} 2024-09-18T00:05:26.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:05:26.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:09 2024-09-18T00:05:26.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:05:26.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:05:26.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:05:26.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:05:26.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:24' 2024-09-18T00:05:26.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:05:26.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:05:26.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:05:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70303744.0,"last_snapshot_bytes":70303744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"} 2024-09-18T00:05:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:05:26.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:26.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:05:26.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:26.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:05:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:05:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:05:26.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:05:26.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:26.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+replaying '2024-09-18 00:05:09' 2024-09-18T00:05:26.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:26.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:26.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:26.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:26.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:05:09' 2024-09-18T00:05:26.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:26.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:26.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:05:26.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69890048.0,"last_snapshot_bytes":69890048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:24image-primary43a58ab54-d559-485b-ad84-79ecd4d5dae7up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70303744.0,"last_snapshot_bytes":70303744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:24image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70090752.0,"last_snapshot_bytes":70090752,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617797,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:24image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63799296.0,"last_snapshot_bytes":63799296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:24image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617810,"remote_snapshot_timestamp":1726617810,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:24image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72302592.0,"last_snapshot_bytes":72302592,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:24image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:24' 2024-09-18T00:05:26.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:05:26.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:05:26.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:26.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:05:26.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:05:26.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:26.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:05:26.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:26.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:26.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:26.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary4 2024-09-18T00:05:26.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:05:26.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:26.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:26.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary4 2024-09-18T00:05:26.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary4 2024-09-18T00:05:28.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:05:28.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary4 2024-09-18T00:05:28.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:28.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:28.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:28.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:28.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:05:28.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary4 2024-09-18T00:05:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary4 2024-09-18T00:05:28.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:28.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:05:28.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary5 2024-09-18T00:05:28.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:05:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+replaying 2024-09-18T00:05:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:28.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:05:29.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+replaying '' '' 2024-09-18T00:05:29.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:29.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:29.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:29.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:29.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:29.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:29.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:29.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:29.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:05:29.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:05:29.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9d12657b-7828-4919-923a-59e2a9847f2c 2024-09-18T00:05:29.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:05:29.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70090752.0,"last_snapshot_bytes":70090752,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617797,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"} 2024-09-18T00:05:29.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:05:29.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:09 2024-09-18T00:05:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:05:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:05:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:05:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:05:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:29' 2024-09-18T00:05:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:05:29.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:05:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:05:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70090752.0,"last_snapshot_bytes":70090752,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617797,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"} 2024-09-18T00:05:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:05:29.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:05:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:29.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:05:29.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:05:29.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:05:29.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:05:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+replaying '2024-09-18 00:05:09' 2024-09-18T00:05:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:05:09' 2024-09-18T00:05:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:29.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:05:29.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK642544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69890048.0,"last_snapshot_bytes":69890048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:29image-primary59d12657b-7828-4919-923a-59e2a9847f2cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70090752.0,"last_snapshot_bytes":70090752,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617797,"remote_snapshot_timestamp":1726617797,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:29image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63799296.0,"last_snapshot_bytes":63799296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:29image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617810,"remote_snapshot_timestamp":1726617810,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:29image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72302592.0,"last_snapshot_bytes":72302592,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:29image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:29' 2024-09-18T00:05:29.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:05:29.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:05:29.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:29.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:05:30.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:05:30.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:30.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:05:30.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary5 2024-09-18T00:05:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:05:30.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:30.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary5 2024-09-18T00:05:30.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary5 2024-09-18T00:05:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:05:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary5 2024-09-18T00:05:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:05:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary5 2024-09-18T00:05:32.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:32.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary5 2024-09-18T00:05:32.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:32.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:05:32.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary6 2024-09-18T00:05:32.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:32.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:32.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:05:32.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:05:32.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+replaying 2024-09-18T00:05:32.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:32.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:32.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:05:32.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:32.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:32.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:32.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:32.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:05:33.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+replaying '' '' 2024-09-18T00:05:33.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:33.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:33.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:05:33.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:33.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:33.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:33.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:33.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:33.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:05:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:05:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 631249db-2f37-4574-b5c1-3ddb41b94859 2024-09-18T00:05:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:05:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63799296.0,"last_snapshot_bytes":63799296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"} 2024-09-18T00:05:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:05:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:09 2024-09-18T00:05:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:05:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:05:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:05:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:05:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:29' 2024-09-18T00:05:33.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:05:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:05:33.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:05:33.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63799296.0,"last_snapshot_bytes":63799296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"} 2024-09-18T00:05:33.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:05:33.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:33.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:05:33.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:33.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:05:33.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:05:33.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:05:33.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:05:33.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:33.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+replaying '2024-09-18 00:05:09' 2024-09-18T00:05:33.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:33.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:33.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:05:33.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:33.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:05:09' 2024-09-18T00:05:33.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:33.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:33.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:05:33.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69890048.0,"last_snapshot_bytes":69890048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:29image-primary6631249db-2f37-4574-b5c1-3ddb41b94859up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63799296.0,"last_snapshot_bytes":63799296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:29image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617810,"remote_snapshot_timestamp":1726617810,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:29image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72302592.0,"last_snapshot_bytes":72302592,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:29image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:29' 2024-09-18T00:05:33.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:05:33.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:05:33.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:33.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:05:33.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:05:33.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:33.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:05:33.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:33.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:33.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:33.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary6 2024-09-18T00:05:33.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:05:33.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:33.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:05:33.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary6 2024-09-18T00:05:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary6 2024-09-18T00:05:34.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:05:34.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary6 2024-09-18T00:05:34.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:34.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:34.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:05:34.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:34.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:05:34.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary6 2024-09-18T00:05:34.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:34.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary6 2024-09-18T00:05:34.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:05:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary7 2024-09-18T00:05:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:05:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:05:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+replaying 2024-09-18T00:05:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:05:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:34.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:34.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:34.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:34.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:05:35.559 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:05:35.560+0000 7f849c00d640 -1 reset not still connected to 0x5629c47aed00 2024-09-18T00:05:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+replaying '' '' 2024-09-18T00:05:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:05:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:35.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:35.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:35.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:05:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:05:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 40e08d00-c76c-475b-a9ba-1e4c6459df5a 2024-09-18T00:05:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:05:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617810,"remote_snapshot_timestamp":1726617810,"replay_state":"idle"} 2024-09-18T00:05:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:05:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:09 2024-09-18T00:05:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:05:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:05:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:05:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:05:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:34' 2024-09-18T00:05:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:05:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:05:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:05:36.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617810,"remote_snapshot_timestamp":1726617810,"replay_state":"idle"} 2024-09-18T00:05:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:05:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:36.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:05:36.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:36.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:05:36.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:05:36.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:05:36.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:05:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:36.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+replaying '2024-09-18 00:05:09' 2024-09-18T00:05:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:05:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:05:09' 2024-09-18T00:05:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:05:36.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69890048.0,"last_snapshot_bytes":69890048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:34image-primary740e08d00-c76c-475b-a9ba-1e4c6459df5aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617810,"remote_snapshot_timestamp":1726617810,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:34image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72302592.0,"last_snapshot_bytes":72302592,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:34image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:34' 2024-09-18T00:05:36.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:05:36.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:05:36.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:36.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:05:36.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:05:36.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:36.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:05:36.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary7 2024-09-18T00:05:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:05:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:05:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary7 2024-09-18T00:05:36.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary7 2024-09-18T00:05:37.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:05:37.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary7 2024-09-18T00:05:37.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:37.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:37.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:05:37.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:37.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:05:37.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary7 2024-09-18T00:05:37.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:37.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary7 2024-09-18T00:05:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:05:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary8 2024-09-18T00:05:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:05:37.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:05:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+replaying 2024-09-18T00:05:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:05:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+replaying '' '' 2024-09-18T00:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:38.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:38.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:38.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:05:38.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:05:38.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 11e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7e 2024-09-18T00:05:38.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:05:38.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72302592.0,"last_snapshot_bytes":72302592,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"} 2024-09-18T00:05:38.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:05:38.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:09 2024-09-18T00:05:38.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:05:38.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:05:38.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:05:38.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:05:38.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:34' 2024-09-18T00:05:38.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:05:38.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:05:38.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:05:38.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72302592.0,"last_snapshot_bytes":72302592,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"} 2024-09-18T00:05:38.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:05:38.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:38.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:05:38.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:38.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:05:38.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:05:38.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:05:38.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:05:38.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:38.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+replaying '2024-09-18 00:05:09' 2024-09-18T00:05:38.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:38.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:38.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:05:38.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:38.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:05:09' 2024-09-18T00:05:38.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:38.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:38.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:05:38.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK342544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69890048.0,"last_snapshot_bytes":69890048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:34image-primary811e7d7cf-3a96-4a5e-8eb6-9b8f1c013f7eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72302592.0,"last_snapshot_bytes":72302592,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617802,"remote_snapshot_timestamp":1726617802,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:34image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:34' 2024-09-18T00:05:38.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:05:38.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:05:38.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:09' 2024-09-18T00:05:38.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:05:38.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:05:38.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:38.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:05:38.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary8 2024-09-18T00:05:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:05:38.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:38.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:05:38.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary8 2024-09-18T00:05:38.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary8 2024-09-18T00:05:39.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary8 2024-09-18T00:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:05:39.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary8 2024-09-18T00:05:39.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:39.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary8 2024-09-18T00:05:39.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:39.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:05:39.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary9 2024-09-18T00:05:39.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:39.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:05:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:05:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+replaying 2024-09-18T00:05:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:05:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:05:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+replaying '' '' 2024-09-18T00:05:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:05:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:40.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:40.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:40.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:40.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:05:41.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:05:41.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 91a54e9f-7ef4-41db-964a-c84ca41a65ce 2024-09-18T00:05:41.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:05:41.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"} 2024-09-18T00:05:41.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:05:41.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:39 2024-09-18T00:05:41.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:05:41.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:05:41.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:05:41.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:05:41.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:39' 2024-09-18T00:05:41.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:05:41.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:05:41.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:05:41.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"} 2024-09-18T00:05:41.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:05:41.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:41.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:05:41.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:41.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:05:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:05:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:05:41.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:05:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:39' 2024-09-18T00:05:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+replaying '2024-09-18 00:05:39' 2024-09-18T00:05:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:05:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:05:39' 2024-09-18T00:05:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:41.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:05:41.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69890048.0,"last_snapshot_bytes":69890048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:39image-primary991a54e9f-7ef4-41db-964a-c84ca41a65ceup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:39' 2024-09-18T00:05:41.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:05:41.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:05:41.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:39' 2024-09-18T00:05:41.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:05:41.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:05:41.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:41.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:05:41.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:41.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:41.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:41.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary9 2024-09-18T00:05:41.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:05:41.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:41.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:05:41.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary9 2024-09-18T00:05:41.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary9 2024-09-18T00:05:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:05:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary9 2024-09-18T00:05:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:05:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:05:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary9 2024-09-18T00:05:42.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:42.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary9 2024-09-18T00:05:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary10 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+replaying 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:42.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:05:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+replaying '' '' 2024-09-18T00:05:43.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:43.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:43.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:05:43.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:43.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:05:43.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:05:43.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:43.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:43.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:05:43.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:05:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 527eda38-0231-4aa6-80fd-b8db0934cf7b 2024-09-18T00:05:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:05:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69890048.0,"last_snapshot_bytes":69890048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"} 2024-09-18T00:05:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:05:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:39 2024-09-18T00:05:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:05:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:05:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:05:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:05:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:05:39' 2024-09-18T00:05:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:05:43.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:05:43.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:05:43.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69890048.0,"last_snapshot_bytes":69890048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"} 2024-09-18T00:05:43.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:05:43.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:43.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:05:43.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:43.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:05:43.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:05:43.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:05:43.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:05:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:39' 2024-09-18T00:05:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+replaying '2024-09-18 00:05:39' 2024-09-18T00:05:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:05:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:05:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:05:39' 2024-09-18T00:05:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:05:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:05:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:05:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10527eda38-0231-4aa6-80fd-b8db0934cf7bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69890048.0,"last_snapshot_bytes":69890048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726617801,"remote_snapshot_timestamp":1726617801,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:05:39' 2024-09-18T00:05:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:05:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:05:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:05:39' 2024-09-18T00:05:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:05:43.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:05:43.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:05:43.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:05:43.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:05:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary10 2024-09-18T00:05:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:05:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:05:43.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary10 2024-09-18T00:05:43.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary10 2024-09-18T00:05:44.076 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:05:44.076+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:05:44.076 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:05:44.076+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:05:44.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:05:44.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary10 2024-09-18T00:05:44.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:44.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:44.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:05:44.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:05:44.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:05:44.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary10 2024-09-18T00:05:44.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary10 2024-09-18T00:05:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..10} 2024-09-18T00:05:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS=() 2024-09-18T00:05:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS=() 2024-09-18T00:05:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS=() 2024-09-18T00:05:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS=() 2024-09-18T00:05:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RET=0 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary1 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary1 snapshot 10G 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary1 10G 2024-09-18T00:05:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:05:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:05:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:05:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:44.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary1 2024-09-18T00:05:45.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:05:45.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:05:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary1 snapshot 2024-09-18T00:05:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary1 snapshot 2024-09-18T00:05:45.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:05:45.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary1 2024-09-18T00:05:45.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary1': 2024-09-18T00:05:45.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:05:45.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:05:45.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:05:45.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 19a940d6b737 2024-09-18T00:05:45.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.19a940d6b737 2024-09-18T00:05:45.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:05:45.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:05:45.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:05:45.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:05:45.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:05:45 2024 2024-09-18T00:05:45.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:05:45 2024 2024-09-18T00:05:45.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:05:45 2024 2024-09-18T00:05:45.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:05:45.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:05:45.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: e748bb2a-a5aa-46c1-b88f-a4adee5d24c6 2024-09-18T00:05:45.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:05:45.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:05:45.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:05:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd0 2024-09-18T00:05:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:05:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd0 2024-09-18T00:05:45.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:05:45.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:05:45.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:05:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: ad924c87-7b2f-4795-8118-9d8aa06b8e5c 2024-09-18T00:05:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:05:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:05:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:05:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:05:46.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:05:46.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:05:46.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary1 2024-09-18T00:05:46.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd0 test-primary1 2024-09-18T00:05:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:05:46.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary1 2024-09-18T00:05:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:05:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary1 2024-09-18T00:05:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:05:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary1 2024-09-18T00:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary2 2024-09-18T00:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary1 2024-09-18T00:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary1 2024-09-18T00:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary2 snapshot 10G 2024-09-18T00:05:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:05:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:46.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:05:46.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary2 10G 2024-09-18T00:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:05:46.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:05:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary2 2024-09-18T00:05:46.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:05:46.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:46.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:05:46.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:46.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary2 snapshot 2024-09-18T00:05:46.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:46.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:46.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:46.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:46.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary2 snapshot 2024-09-18T00:05:47.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:05:47.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 664 2024-09-18T00:05:47.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:47.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary2 2024-09-18T00:05:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary2': 2024-09-18T00:05:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:05:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:05:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:05:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 19b9782f14c5 2024-09-18T00:05:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.19b9782f14c5 2024-09-18T00:05:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:05:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:05:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:05:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:05:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:05:46 2024 2024-09-18T00:05:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:05:46 2024 2024-09-18T00:05:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:05:46 2024 2024-09-18T00:05:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:05:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:05:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: f00bdbdc-f2bd-4210-80a6-7c7769eada40 2024-09-18T00:05:47.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:05:47.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:05:47.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:05:47.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd1 2024-09-18T00:05:47.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:05:47.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd1 2024-09-18T00:05:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:05:47.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:05:47.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:05:47.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: dbf841ba-8fef-4411-8b2d-e91cfa506271 2024-09-18T00:05:47.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:05:47.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:05:47.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:47.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:05:47.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:05:47.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:05:47.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary1/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary1' 2024-09-18T00:05:47.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:05:47.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:05:47.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:47.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary2 2024-09-18T00:05:47.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd1 test-primary2 2024-09-18T00:05:47.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:05:47.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary2 2024-09-18T00:05:47.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:05:47.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary2 2024-09-18T00:05:47.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:05:47.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:47.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary2 2024-09-18T00:05:47.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:05:47.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary3 2024-09-18T00:05:47.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary3 snapshot 10G 2024-09-18T00:05:47.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:47.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:47.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:47.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:47.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary2 2024-09-18T00:05:47.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:47.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:47.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:47.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:47.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:47.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:05:47.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:47.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary2 2024-09-18T00:05:47.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:47.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary3 10G 2024-09-18T00:05:47.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:47.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:05:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:05:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:05:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary3 2024-09-18T00:05:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:05:47.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:47.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:47.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:47.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:05:47.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:05:47.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:47.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:05:47.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:47.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary3 snapshot 2024-09-18T00:05:47.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:47.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:47.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:47.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:47.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary3 snapshot 2024-09-18T00:05:48.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:05:48.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary2/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary2' 2024-09-18T00:05:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:05:48.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 666 2024-09-18T00:05:48.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:48.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary3 2024-09-18T00:05:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary3': 2024-09-18T00:05:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:05:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:05:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:05:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 19c6d506da63 2024-09-18T00:05:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.19c6d506da63 2024-09-18T00:05:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:05:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:05:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:05:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:05:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:05:47 2024 2024-09-18T00:05:48.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:05:47 2024 2024-09-18T00:05:48.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:05:47 2024 2024-09-18T00:05:48.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:05:48.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:05:48.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 49280c03-8d1c-4c44-b8cc-dc519e871b86 2024-09-18T00:05:48.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:05:48.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:05:48.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:05:48.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd2 2024-09-18T00:05:48.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:05:48.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd2 2024-09-18T00:05:48.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:05:48.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:05:48.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:05:48.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 77ef3712-3062-40ff-8f7e-95821721f54c 2024-09-18T00:05:48.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:05:48.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:05:48.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:48.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:05:48.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:05:48.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:05:48.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:05:48.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:48.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary3 2024-09-18T00:05:48.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd2 test-primary3 2024-09-18T00:05:48.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:05:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary3 2024-09-18T00:05:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:05:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary3 2024-09-18T00:05:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:05:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary3 2024-09-18T00:05:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:05:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary4 2024-09-18T00:05:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary4 snapshot 10G 2024-09-18T00:05:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary3 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary3 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary4 10G 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:05:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:05:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:05:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary4 2024-09-18T00:05:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:05:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:05:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:05:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:05:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:49.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary4 snapshot 2024-09-18T00:05:49.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:49.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:49.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:49.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:49.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary4 snapshot 2024-09-18T00:05:49.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary3/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary3' 2024-09-18T00:05:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 668 2024-09-18T00:05:49.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:05:49.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:49.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary4 2024-09-18T00:05:49.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary4': 2024-09-18T00:05:49.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:05:49.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:05:49.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:05:49.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 19d5558c1f6d 2024-09-18T00:05:49.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.19d5558c1f6d 2024-09-18T00:05:49.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:05:49.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:05:49.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:05:49.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:05:49.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:05:49 2024 2024-09-18T00:05:49.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:05:49 2024 2024-09-18T00:05:49.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:05:49 2024 2024-09-18T00:05:49.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:05:49.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:05:49.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3 2024-09-18T00:05:49.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:05:49.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:05:49.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:05:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd3 2024-09-18T00:05:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:05:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd3 2024-09-18T00:05:49.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: f245f59b-cdd1-4cc9-8c90-e30281ba022f 2024-09-18T00:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:05:49.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:05:49.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:05:49.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:05:49.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:05:49.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:49.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary4 2024-09-18T00:05:49.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd3 test-primary4 2024-09-18T00:05:49.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:05:49.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary4 2024-09-18T00:05:49.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:05:49.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary4 2024-09-18T00:05:49.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:05:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary4 2024-09-18T00:05:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:05:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary5 2024-09-18T00:05:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary5 snapshot 10G 2024-09-18T00:05:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:49.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:05:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary5 10G 2024-09-18T00:05:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:05:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:05:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:05:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:05:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:05:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary5 2024-09-18T00:05:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary4 2024-09-18T00:05:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary4 2024-09-18T00:05:50.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:05:50.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:50.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:05:50.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:50.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary5 snapshot 2024-09-18T00:05:50.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:50.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:50.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:50.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:50.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary5 snapshot 2024-09-18T00:05:50.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:05:50.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary4/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary4' 2024-09-18T00:05:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:05:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:50.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:05:50.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:05:50.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 670 2024-09-18T00:05:50.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:50.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary5 2024-09-18T00:05:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary5': 2024-09-18T00:05:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:05:50.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:05:50.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:05:50.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 19e6664466b8 2024-09-18T00:05:50.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.19e6664466b8 2024-09-18T00:05:50.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:05:50.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:05:50.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:05:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:05:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:05:50 2024 2024-09-18T00:05:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:05:50 2024 2024-09-18T00:05:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:05:50 2024 2024-09-18T00:05:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:05:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:05:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: fac94502-f15a-4e8a-b812-20cd1df947cf 2024-09-18T00:05:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:05:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:05:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:05:50.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd4 2024-09-18T00:05:50.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:05:50.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd4 2024-09-18T00:05:50.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:05:50.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:05:50.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:05:50.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: fe5d7a53-3d9a-4732-b81c-8ec375d3f581 2024-09-18T00:05:50.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:05:50.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:05:50.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:50.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:05:50.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:05:50.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:05:50.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:05:50.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:50.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary5 2024-09-18T00:05:50.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd4 test-primary5 2024-09-18T00:05:50.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:05:50.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary5 2024-09-18T00:05:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:05:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary5 2024-09-18T00:05:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:05:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:05:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary5 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary6 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary6 snapshot 10G 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary5 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary5 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary6 10G 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:05:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary6 2024-09-18T00:05:50.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:05:50.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:50.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:05:50.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:50.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary6 snapshot 2024-09-18T00:05:50.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:50.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:50.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:05:50.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:50.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary6 snapshot 2024-09-18T00:05:51.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:05:51.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary5/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary5' 2024-09-18T00:05:51.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:05:51.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:05:51.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 673 2024-09-18T00:05:51.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:51.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary6 2024-09-18T00:05:51.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary6': 2024-09-18T00:05:51.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:05:51.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:05:51.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:05:51.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 19f8937661d7 2024-09-18T00:05:51.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.19f8937661d7 2024-09-18T00:05:51.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:05:51.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:05:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:05:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:05:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:05:50 2024 2024-09-18T00:05:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:05:50 2024 2024-09-18T00:05:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:05:50 2024 2024-09-18T00:05:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:05:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:05:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: e7927f4e-32c4-4eaa-be90-8922d3a6eb2f 2024-09-18T00:05:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:05:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:05:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:05:51.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd5 2024-09-18T00:05:51.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:05:51.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd5 2024-09-18T00:05:51.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:05:51.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:05:51.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:05:51.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: ab1c0051-8a60-409c-9c40-7230edd69842 2024-09-18T00:05:51.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:05:51.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:05:51.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:51.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:05:51.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:05:51.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:05:52.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:05:52.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:52.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary6 2024-09-18T00:05:52.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd5 test-primary6 2024-09-18T00:05:52.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 672 2024-09-18T00:05:52.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:52.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:05:52.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary6 2024-09-18T00:05:52.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:05:52.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary6 2024-09-18T00:05:52.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:52.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:52.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:05:52.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:05:52.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary7 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary6 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary7 snapshot 10G 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:05:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary6 2024-09-18T00:05:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:05:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary6 2024-09-18T00:05:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary7 10G 2024-09-18T00:05:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:05:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:05:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:05:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:05:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary7 2024-09-18T00:05:52.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:05:52.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:05:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary7 snapshot 2024-09-18T00:05:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:05:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary7 snapshot 2024-09-18T00:05:52.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:52.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:05:52.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:52.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:52.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:52.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:05:52.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:05:52.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary6/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary6' 2024-09-18T00:05:53.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 675 2024-09-18T00:05:53.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:53.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 676 2024-09-18T00:05:53.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:53.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:05:53.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary7 2024-09-18T00:05:53.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary7': 2024-09-18T00:05:53.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:05:53.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:05:53.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:05:53.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1a0bd2e5b0c7 2024-09-18T00:05:53.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1a0bd2e5b0c7 2024-09-18T00:05:53.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:05:53.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:05:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:05:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:05:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:05:52 2024 2024-09-18T00:05:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:05:52 2024 2024-09-18T00:05:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:05:52 2024 2024-09-18T00:05:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:05:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:05:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 5cbf0aba-1bfb-4812-b772-7406d18bca14 2024-09-18T00:05:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:05:53.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:05:53.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:05:53.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:53.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:05:53.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:53.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:53.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:53.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:05:53.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd6 2024-09-18T00:05:53.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:05:53.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd6 2024-09-18T00:05:53.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:05:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:05:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:05:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: a778991e-b109-487d-9fbb-2c123fb28f79 2024-09-18T00:05:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:05:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:05:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:05:53.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:05:53.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:05:53.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:05:53.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:53.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary7 2024-09-18T00:05:53.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd6 test-primary7 2024-09-18T00:05:53.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:05:53.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary7 2024-09-18T00:05:53.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:05:53.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary7 2024-09-18T00:05:53.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:05:53.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:53.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary7 2024-09-18T00:05:53.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:05:53.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:53.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary8 2024-09-18T00:05:53.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:53.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary8 snapshot 10G 2024-09-18T00:05:53.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:05:53.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:53.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary7 2024-09-18T00:05:53.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:53.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:05:53.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:53.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary7 2024-09-18T00:05:53.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:53.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:53.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:53.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:05:53.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:53.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:53.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary8 10G 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:05:53.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:05:53.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:53.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary8 2024-09-18T00:05:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:05:54.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:54.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:05:54.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:54.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary8 snapshot 2024-09-18T00:05:54.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:54.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:54.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:05:54.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:54.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary8 snapshot 2024-09-18T00:05:54.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:05:54.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary7/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary7' 2024-09-18T00:05:54.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 678 2024-09-18T00:05:54.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:54.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 680 2024-09-18T00:05:54.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:54.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:54.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:05:54.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:54.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:54.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:54.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:05:55.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:55.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:05:55.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:55.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:55.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:05:55.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:05:55.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 679 2024-09-18T00:05:55.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:55.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:05:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary8 2024-09-18T00:05:55.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary8': 2024-09-18T00:05:55.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:05:55.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:05:55.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:05:55.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1a20661b154a 2024-09-18T00:05:55.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1a20661b154a 2024-09-18T00:05:55.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:05:55.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:05:55.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:05:55.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:05:55.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:05:54 2024 2024-09-18T00:05:55.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:05:54 2024 2024-09-18T00:05:55.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:05:54 2024 2024-09-18T00:05:55.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:05:55.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:05:55.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 2846127d-142c-45b9-b096-a07f221235bd 2024-09-18T00:05:55.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:05:55.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:05:55.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:05:55.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd7 2024-09-18T00:05:55.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:05:55.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd7 2024-09-18T00:05:55.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:05:55.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:05:55.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:05:55.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: bdb1707e-aed3-4639-9de0-fd677f0b1e57 2024-09-18T00:05:55.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:05:55.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:05:55.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:55.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:05:55.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:05:55.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:05:55.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:05:55.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:55.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary8 2024-09-18T00:05:55.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd7 test-primary8 2024-09-18T00:05:55.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:05:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary8 2024-09-18T00:05:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:05:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary8 2024-09-18T00:05:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:05:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:05:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary8 2024-09-18T00:05:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:05:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:05:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary9 2024-09-18T00:05:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:55.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary9 snapshot 10G 2024-09-18T00:05:55.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:55.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:05:55.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:05:55.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:55.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary8 2024-09-18T00:05:55.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:55.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:55.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary8 2024-09-18T00:05:55.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:55.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:05:55.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:55.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:55.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:05:55.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:55.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary9 10G 2024-09-18T00:05:55.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:05:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:05:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:05:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:05:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary9 2024-09-18T00:05:56.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:56.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:05:56.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:56.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:56.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:05:56.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:05:56.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:05:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:05:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary9 snapshot 2024-09-18T00:05:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:05:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary9 snapshot 2024-09-18T00:05:56.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:56.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:05:56.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:56.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:56.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:05:56.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:05:56.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:05:56.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary8/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary8' 2024-09-18T00:05:56.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 682 2024-09-18T00:05:56.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:57.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 683 2024-09-18T00:05:57.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:57.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:57.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:05:57.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:57.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:57.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:05:57.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:05:57.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:05:57.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:57.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:05:57.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:57.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:57.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:05:57.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:05:57.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 684 2024-09-18T00:05:57.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:57.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary9 2024-09-18T00:05:57.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary9': 2024-09-18T00:05:57.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:05:57.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1a3090c8e872 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1a3090c8e872 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:05:56 2024 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:05:56 2024 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:05:56 2024 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:05:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: bcd32a3a-52fc-4137-aa0a-509317d6523f 2024-09-18T00:05:57.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:05:57.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:05:57.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:05:57.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd8 2024-09-18T00:05:57.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:05:57.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd8 2024-09-18T00:05:57.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:05:57.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:05:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:05:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 7979da39-ef18-42a3-8afe-97a614143b51 2024-09-18T00:05:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:05:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:05:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:05:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:05:57.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:05:57.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:05:57.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:57.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary9 2024-09-18T00:05:57.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd8 test-primary9 2024-09-18T00:05:57.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:05:58.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary9 2024-09-18T00:05:58.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:05:58.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary9 2024-09-18T00:05:58.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:05:58.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:05:58.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:58.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary9 2024-09-18T00:05:58.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary10 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary10 snapshot 10G 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary9 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary9 2024-09-18T00:05:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:05:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:05:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary10 10G 2024-09-18T00:05:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:58.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:05:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:05:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:05:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:05:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:05:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary10 2024-09-18T00:05:58.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:05:58.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary9/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary9' 2024-09-18T00:05:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:05:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:05:58.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:05:58.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:05:58.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:58.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:05:58.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:05:58.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary10 snapshot 2024-09-18T00:05:58.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:58.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:58.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:05:58.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:05:58.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary10 snapshot 2024-09-18T00:05:58.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 686 2024-09-18T00:05:58.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 685 2024-09-18T00:05:58.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:58.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:59.004 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.001+0000 7fd4c158b640 -1 reset not still connected to 0x564358707d40 2024-09-18T00:05:59.004 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.001+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a340 2024-09-18T00:05:59.004 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.001+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T00:05:59.004 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.001+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T00:05:59.005 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.001+0000 7fd4c158b640 -1 reset not still connected to 0x5643595bdc70 2024-09-18T00:05:59.005 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.001+0000 7fd4c158b640 -1 reset not still connected to 0x56435b71c270 2024-09-18T00:05:59.005 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.005+0000 7fd4c158b640 -1 reset not still connected to 0x56435ba7fa00 2024-09-18T00:05:59.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 688 2024-09-18T00:05:59.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 689 2024-09-18T00:05:59.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:59.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 692 2024-09-18T00:05:59.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:59.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:05:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:05:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:05:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:05:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:05:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:05:59.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:05:59.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:05:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary10 2024-09-18T00:05:59.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary10': 2024-09-18T00:05:59.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:05:59.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:05:59.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:05:59.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1a4ac170d541 2024-09-18T00:05:59.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1a4ac170d541 2024-09-18T00:05:59.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:05:59.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:05:59.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:05:59.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:05:59.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:05:58 2024 2024-09-18T00:05:59.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:05:58 2024 2024-09-18T00:05:59.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:05:58 2024 2024-09-18T00:05:59.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:05:59.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:05:59.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 02b36501-4523-474d-b4d9-f59723d6b12e 2024-09-18T00:05:59.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:05:59.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:05:59.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:05:59.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd9 2024-09-18T00:05:59.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:05:59.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd9 2024-09-18T00:05:59.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:05:59.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:05:59.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:05:59.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 5d306bbc-2881-4897-a852-6571954974bb 2024-09-18T00:05:59.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:05:59.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:05:59.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:59.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:05:59.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:05:59.744 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.737+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a340 2024-09-18T00:05:59.744 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.737+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T00:05:59.749 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.737+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T00:05:59.753 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.745+0000 7fd4c158b640 -1 reset not still connected to 0x56435b71c270 2024-09-18T00:05:59.753 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.749+0000 7fd4c158b640 -1 reset not still connected to 0x56435b781790 2024-09-18T00:05:59.753 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:05:59.749+0000 7fd4c158b640 -1 reset not still connected to 0x56435ba7fa00 2024-09-18T00:05:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:05:59.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:05:59.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:05:59.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary10 2024-09-18T00:05:59.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd9 test-primary10 2024-09-18T00:06:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:06:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary10 2024-09-18T00:06:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:06:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary10 2024-09-18T00:06:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:06:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary10 2024-09-18T00:06:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:06:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:06:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 52263 2024-09-18T00:06:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary10 2024-09-18T00:06:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary10 2024-09-18T00:06:00.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:00.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:06:00.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:00.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:00.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:06:00.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:06:00.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 690 2024-09-18T00:06:00.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:06:00.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary10/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary10' 2024-09-18T00:06:00.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:06:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:06:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:06:01.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 693 2024-09-18T00:06:01.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:01.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:01.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:06:01.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:01.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:01.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:06:01.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:06:01.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 695 2024-09-18T00:06:01.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:01.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:06:01.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:01.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:01.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:06:01.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:06:01.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:02.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 694 2024-09-18T00:06:02.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:02.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:06:02.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:02.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:06:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:06:02.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:06:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:02.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:02.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:06:02.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:06:02.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 696 2024-09-18T00:06:02.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:02.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:06:02.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:02.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:02.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:06:02.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:06:02.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:03.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 698 2024-09-18T00:06:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 697 2024-09-18T00:06:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:06:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:06:03.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:06:03.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:03.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:04.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 699 2024-09-18T00:06:04.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 701 2024-09-18T00:06:04.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:04.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:06:04.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:04.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:04.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:06:04.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:06:04.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 700 2024-09-18T00:06:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:04.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:04.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:04.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:04.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:04.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:04.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:04.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:04.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:06:05.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:05.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:06:05.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:05.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:05.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:06:05.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:06:05.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:05.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:06:05.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:05.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:05.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:06:05.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:06:05.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 702 2024-09-18T00:06:05.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:06.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 704 2024-09-18T00:06:06.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 703 2024-09-18T00:06:06.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:06.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:06.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:06:06.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:06.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:06.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:06:06.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:06:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:06:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:06:06.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:06:06.276 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:06:06.280+0000 7f849c00d640 -1 reset not still connected to 0x5629aeef5450 2024-09-18T00:06:06.277 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:06:06.280+0000 7f849c00d640 -1 reset not still connected to 0x5629c4a87d40 2024-09-18T00:06:06.277 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:06:06.280+0000 7f849c00d640 -1 reset not still connected to 0x5629cc435d40 2024-09-18T00:06:07.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 705 2024-09-18T00:06:07.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 706 2024-09-18T00:06:07.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:07.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:07.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:07.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:06:07.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:07.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:07.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:06:07.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:06:07.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:07.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:06:07.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:07.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:07.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:06:07.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:06:08.205 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:06:08.200+0000 7f849c00d640 -1 reset not still connected to 0x5629aeef5450 2024-09-18T00:06:08.206 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:06:08.200+0000 7f849c00d640 -1 reset not still connected to 0x5629c4a87d40 2024-09-18T00:06:08.206 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:06:08.200+0000 7f849c00d640 -1 reset not still connected to 0x5629cc435d40 2024-09-18T00:06:08.206 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:06:08.200+0000 7f849c00d640 -1 reset not still connected to 0x5629c74a6ea0 2024-09-18T00:06:08.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 708 2024-09-18T00:06:08.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 707 2024-09-18T00:06:08.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:08.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:08.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:06:08.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:08.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:08.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:06:08.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:06:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:09.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:09.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:09.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:09.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:09.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:06:09.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 711 2024-09-18T00:06:09.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 709 2024-09-18T00:06:09.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:09.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:06:09.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:09.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:09.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:06:09.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:06:09.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 710 2024-09-18T00:06:09.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:09.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:09.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:10.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:10.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:06:10.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:10.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:10.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:06:10.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:06:10.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:10.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:06:10.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:10.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:10.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:06:10.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:06:10.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 712 2024-09-18T00:06:10.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:11.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:11.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:06:11.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:11.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:11.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:06:11.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:06:11.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:11.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:06:11.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:11.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:11.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:06:11.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:06:11.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 713 2024-09-18T00:06:11.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 714 2024-09-18T00:06:11.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:12.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 716 2024-09-18T00:06:12.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:12.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 715 2024-09-18T00:06:12.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:12.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:12.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:06:12.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:12.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:12.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:06:12.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:06:12.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:12.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:06:12.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:12.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:12.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:06:12.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:06:12.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:12.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:06:12.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:12.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:12.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:06:12.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:06:13.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:13.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:06:13.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:13.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:13.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:06:13.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:06:13.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 718 2024-09-18T00:06:13.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 717 2024-09-18T00:06:13.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:13.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:14.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 719 2024-09-18T00:06:14.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:14.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 721 2024-09-18T00:06:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 720 2024-09-18T00:06:14.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:14.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:14.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:14.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:14.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:14.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:06:14.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:14.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:14.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:14.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:06:14.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:14.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:14.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:06:14.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:06:15.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:15.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:06:15.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:15.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:15.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:06:15.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:06:15.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:15.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:06:15.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:15.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:15.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:06:15.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:06:15.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 722 2024-09-18T00:06:15.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:16.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 723 2024-09-18T00:06:16.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 724 2024-09-18T00:06:16.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:16.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:16.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:06:16.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:16.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:16.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:06:16.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:06:16.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:16.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:06:16.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:16.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:16.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:06:16.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:06:17.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 726 2024-09-18T00:06:17.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 725 2024-09-18T00:06:17.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:17.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:17.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:17.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:06:17.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:17.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:17.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:06:17.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:06:17.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:17.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:06:17.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:17.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:17.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:06:17.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:06:17.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:17.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:06:17.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:17.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:17.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:06:17.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:06:18.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 727 2024-09-18T00:06:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:06:18.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:06:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:06:18.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 728 2024-09-18T00:06:18.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:18.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:19.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 731 2024-09-18T00:06:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:19.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:06:19.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 730 2024-09-18T00:06:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:06:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:06:19.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:06:19.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 729 2024-09-18T00:06:19.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:19.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:19.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:06:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:06:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:06:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:06:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:06:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:06:20.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 732 2024-09-18T00:06:20.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:21.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:21.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:06:21.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:21.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:21.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:06:21.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:06:21.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 733 2024-09-18T00:06:21.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 734 2024-09-18T00:06:21.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:21.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:06:21.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:21.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:21.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:06:21.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:06:21.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:21.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:22.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 736 2024-09-18T00:06:22.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:22.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 735 2024-09-18T00:06:22.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:22.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:22.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:22.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:22.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:06:22.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:06:22.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:06:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:06:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:06:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:06:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:06:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:06:22.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:06:23.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 738 2024-09-18T00:06:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:06:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:06:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:06:23.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 737 2024-09-18T00:06:23.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 739 2024-09-18T00:06:24.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 740 2024-09-18T00:06:24.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 741 2024-09-18T00:06:24.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:24.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:06:24.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:24.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:24.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:06:24.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:06:24.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:24.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:24.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:24.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:24.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:24.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:24.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:24.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:06:24.885 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:06:24.888+0000 7f52d7c6c640 -1 reset not still connected to 0x5578426955f0 2024-09-18T00:06:24.885 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:06:24.888+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f85b0 2024-09-18T00:06:24.886 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:06:24.888+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f89c0 2024-09-18T00:06:24.886 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:06:24.888+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T00:06:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:06:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:06:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:06:25.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:25.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:06:25.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:25.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:25.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:06:25.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:06:25.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 742 2024-09-18T00:06:25.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:26.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 743 2024-09-18T00:06:26.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:26.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:06:26.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:26.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:26.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:06:26.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:06:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 744 2024-09-18T00:06:26.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:26.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:26.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:06:26.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:26.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:26.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:06:26.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:06:26.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:27.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 745 2024-09-18T00:06:27.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:27.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 746 2024-09-18T00:06:27.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:27.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:06:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:06:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:06:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:06:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:06:27.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:06:27.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:27.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:06:27.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:06:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:06:28.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 747 2024-09-18T00:06:28.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:28.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:06:28.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:28.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:28.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:06:28.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:06:28.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 748 2024-09-18T00:06:28.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:28.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:29.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 749 2024-09-18T00:06:29.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:29.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 751 2024-09-18T00:06:29.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 750 2024-09-18T00:06:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:29.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:06:29.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:29.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:06:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:06:29.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:06:30.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:30.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:06:30.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:30.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:30.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:06:30.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:06:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:06:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:06:30.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:06:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 752 2024-09-18T00:06:30.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:31.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 753 2024-09-18T00:06:31.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:31.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:06:31.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:31.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:31.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:06:31.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:06:31.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 754 2024-09-18T00:06:31.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:31.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:31.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:06:31.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:31.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:31.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:06:31.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:06:31.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:32.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 756 2024-09-18T00:06:32.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:32.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 755 2024-09-18T00:06:32.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:32.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:32.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:06:32.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:32.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:32.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:06:32.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:06:32.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:32.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:06:32.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:32.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:32.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:06:32.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:32.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:06:32.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:06:32.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:32.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:32.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:06:32.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:06:33.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 758 2024-09-18T00:06:33.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 757 2024-09-18T00:06:33.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:33.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:06:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:06:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:06:34.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 759 2024-09-18T00:06:34.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 761 2024-09-18T00:06:34.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:34.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 760 2024-09-18T00:06:34.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:34.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:34.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:34.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:34.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:34.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:34.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:34.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:06:34.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:34.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:06:34.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:34.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:34.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:06:34.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:06:35.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 762 2024-09-18T00:06:35.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:35.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:06:35.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:06:35.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:06:35.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:35.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:35.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:06:35.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:35.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:35.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:06:35.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:06:36.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:36.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:06:36.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:36.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:36.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:06:36.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:06:36.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:36.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:06:36.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:36.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:36.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:06:36.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:06:36.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 763 2024-09-18T00:06:36.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 764 2024-09-18T00:06:36.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:36.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:37.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 765 2024-09-18T00:06:37.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:37.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 766 2024-09-18T00:06:37.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:06:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:06:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:06:37.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:37.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:06:37.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:37.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:37.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:06:37.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:06:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:06:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:06:37.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:06:38.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:38.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:06:38.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:38.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:38.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:06:38.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:06:38.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 768 2024-09-18T00:06:38.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 767 2024-09-18T00:06:38.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:38.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:39.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 771 2024-09-18T00:06:39.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 770 2024-09-18T00:06:39.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 769 2024-09-18T00:06:39.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:39.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:39.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:39.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:39.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:39.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:06:39.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:39.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:39.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:06:39.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:39.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:39.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:06:39.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:06:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:40.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:40.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:06:40.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:40.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:40.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:06:40.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:06:40.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:06:40.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 772 2024-09-18T00:06:40.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 773 2024-09-18T00:06:41.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:41.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:06:41.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:41.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:41.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:06:41.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:06:41.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 774 2024-09-18T00:06:41.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:41.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:06:41.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:41.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:41.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:06:41.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:06:41.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:41.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:42.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 776 2024-09-18T00:06:42.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 775 2024-09-18T00:06:42.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:42.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:42.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:42.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:06:42.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:42.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:42.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:06:42.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:06:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:06:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:06:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:06:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:06:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:06:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:06:43.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:43.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:06:43.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:43.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:43.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:06:43.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:06:43.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 778 2024-09-18T00:06:43.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 777 2024-09-18T00:06:43.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:43.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:44.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 780 2024-09-18T00:06:44.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:44.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 779 2024-09-18T00:06:44.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 781 2024-09-18T00:06:44.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:44.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:06:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:06:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:06:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:44.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:06:44.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:45.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:45.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:06:45.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:45.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:45.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:06:45.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:06:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:06:45.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:45.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:45.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:06:45.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:06:45.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 782 2024-09-18T00:06:45.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:46.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:46.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:06:46.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:46.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:46.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:06:46.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:06:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:06:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:06:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:06:46.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 784 2024-09-18T00:06:46.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 783 2024-09-18T00:06:46.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:46.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:47.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 786 2024-09-18T00:06:47.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:47.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 785 2024-09-18T00:06:47.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:47.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:47.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:06:47.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:47.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:47.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:06:47.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:06:47.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:47.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:06:47.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:47.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:47.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:06:47.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:06:47.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:47.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:06:47.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:47.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:47.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:06:47.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:06:48.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 787 2024-09-18T00:06:48.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 788 2024-09-18T00:06:48.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:48.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:48.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:48.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:06:48.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:48.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:48.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:06:48.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:06:49.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 791 2024-09-18T00:06:49.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 790 2024-09-18T00:06:49.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 789 2024-09-18T00:06:49.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:49.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:49.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:49.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:49.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:06:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:06:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:06:49.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:49.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:49.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:49.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:49.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:49.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:06:50.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:50.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:06:50.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:50.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:50.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:06:50.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:06:50.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:50.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:06:50.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:50.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:50.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:06:50.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:06:50.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 792 2024-09-18T00:06:50.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:51.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 794 2024-09-18T00:06:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:06:51.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:51.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:51.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:06:51.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:06:51.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:51.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:06:51.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:51.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:51.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:06:51.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:06:51.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:51.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 793 2024-09-18T00:06:51.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:51.861 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:06:51.863+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a340 2024-09-18T00:06:51.862 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:06:51.863+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T00:06:51.862 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:06:51.863+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T00:06:51.862 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:06:51.863+0000 7fd4c158b640 -1 reset not still connected to 0x5643595bdc70 2024-09-18T00:06:51.862 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:06:51.863+0000 7fd4c158b640 -1 reset not still connected to 0x56435b781790 2024-09-18T00:06:51.862 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:06:51.863+0000 7fd4c158b640 -1 reset not still connected to 0x56435ba7fa00 2024-09-18T00:06:52.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 795 2024-09-18T00:06:52.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 796 2024-09-18T00:06:52.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:52.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:06:52.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:06:52.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:06:53.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:06:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:06:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:06:53.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 798 2024-09-18T00:06:53.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 797 2024-09-18T00:06:53.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:53.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:54.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:06:54.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 800 2024-09-18T00:06:54.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 801 2024-09-18T00:06:54.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 799 2024-09-18T00:06:54.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:54.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:54.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:54.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:54.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:54.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:06:54.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:54.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:54.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:55.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:06:55.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:06:55.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 802 2024-09-18T00:06:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:56.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 803 2024-09-18T00:06:56.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 804 2024-09-18T00:06:56.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:56.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:06:56.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:56.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:56.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:06:56.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:06:56.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:56.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:56.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:06:56.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:56.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:56.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:06:56.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:06:56.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:57.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 806 2024-09-18T00:06:57.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 805 2024-09-18T00:06:57.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:57.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:57.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:57.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:06:57.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:57.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:57.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:06:57.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:06:57.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:57.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:06:57.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:57.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:57.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:06:57.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:06:57.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:57.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:06:57.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:57.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:57.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:06:57.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:06:58.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 807 2024-09-18T00:06:58.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 808 2024-09-18T00:06:58.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:58.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:58.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:58.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:06:58.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:58.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:58.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:06:58.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:06:59.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 811 2024-09-18T00:06:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 810 2024-09-18T00:06:59.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:59.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 809 2024-09-18T00:06:59.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:59.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:06:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:06:59.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:59.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:59.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:06:59.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:06:59.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:06:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:06:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:06:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:06:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:06:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:00.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 812 2024-09-18T00:07:00.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:07:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:07:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:07:00.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:00.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:07:00.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:00.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:00.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:07:00.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:07:01.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:01.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:01.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:07:01.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:07:01.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:01.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:01.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:07:01.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:07:01.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:01.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:01.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:07:01.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:07:01.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 814 2024-09-18T00:07:01.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 813 2024-09-18T00:07:01.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:01.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:02.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 815 2024-09-18T00:07:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 816 2024-09-18T00:07:02.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:02.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:02.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:07:02.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:02.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:02.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:07:02.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:07:02.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:02.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:02.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:07:02.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:02.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:02.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:07:02.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:07:02.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:02.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:07:02.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:02.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:02.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:07:02.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:07:03.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:03.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:07:03.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:03.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:03.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:07:03.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:07:03.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 818 2024-09-18T00:07:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 817 2024-09-18T00:07:03.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:03.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:04.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 821 2024-09-18T00:07:04.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 820 2024-09-18T00:07:04.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:04.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:07:04.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:04.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:04.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:07:04.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:04.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:04.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:07:04.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:04.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:04.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:07:04.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:07:04.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 819 2024-09-18T00:07:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:04.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:04.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:05.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:05.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:07:05.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:05.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:05.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:07:05.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:07:05.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 822 2024-09-18T00:07:05.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:05.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:07:05.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:05.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:05.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:07:05.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:07:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:06.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 823 2024-09-18T00:07:06.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 824 2024-09-18T00:07:06.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:06.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:07:06.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:06.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:06.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:07:06.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:07:06.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:06.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:07:06.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:06.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:06.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:07:06.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:07:06.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:06.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:07.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 825 2024-09-18T00:07:07.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 826 2024-09-18T00:07:07.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:07.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:07.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:07.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:07:07.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:07.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:07.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:07:07.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:07:07.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:07.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:07:07.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:07.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:07.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:07:07.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:07:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:07:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:07:07.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:07:08.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:08.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:07:08.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:08.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:08.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:07:08.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:07:08.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 828 2024-09-18T00:07:08.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 827 2024-09-18T00:07:08.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:08.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:09.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 829 2024-09-18T00:07:09.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 831 2024-09-18T00:07:09.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 830 2024-09-18T00:07:09.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:09.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:07:09.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:09.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:09.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:07:09.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:09.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:09.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:07:09.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:09.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:09.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:07:09.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:07:09.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:09.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:09.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:10.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:10.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:07:10.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:10.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:10.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:07:10.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:07:10.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:07:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:07:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:07:10.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 832 2024-09-18T00:07:10.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:11.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:11.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:07:11.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:11.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:11.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:07:11.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:07:11.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:11.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:07:11.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:11.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:11.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:07:11.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:07:11.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 833 2024-09-18T00:07:11.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 834 2024-09-18T00:07:11.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:11.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:12.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 835 2024-09-18T00:07:12.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:12.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 836 2024-09-18T00:07:12.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:12.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:12.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:07:12.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:12.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:12.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:07:12.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:07:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:07:12.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:12.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:12.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:07:12.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:07:12.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:12.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:07:12.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:12.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:12.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:07:12.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:07:13.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 838 2024-09-18T00:07:13.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:13.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:07:13.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:13.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:13.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:07:13.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:07:13.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 837 2024-09-18T00:07:13.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:13.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:14.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 840 2024-09-18T00:07:14.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 839 2024-09-18T00:07:14.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 841 2024-09-18T00:07:14.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:14.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:14.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:07:14.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:14.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:14.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:07:14.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:07:14.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:14.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:14.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:07:14.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:14.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:14.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:07:14.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:14.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:15.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:15.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:07:15.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:15.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:15.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:07:15.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:07:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:07:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:07:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:07:15.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 842 2024-09-18T00:07:15.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:16.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 843 2024-09-18T00:07:16.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 844 2024-09-18T00:07:16.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:07:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:07:16.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:07:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:07:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:07:16.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:07:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:16.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:17.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 846 2024-09-18T00:07:17.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 845 2024-09-18T00:07:17.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:17.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:17.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:17.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:07:17.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:17.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:17.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:07:17.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:07:17.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:17.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:07:17.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:17.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:17.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:07:17.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:07:17.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:17.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:07:17.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:17.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:17.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:07:17.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:07:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:07:18.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:18.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:18.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:07:18.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:07:18.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 847 2024-09-18T00:07:18.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:18.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 848 2024-09-18T00:07:18.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:18.878 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:07:18.878+0000 7fd4c158b640 -1 reset not still connected to 0x56435ba7fa00 2024-09-18T00:07:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:07:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:19.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:07:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:07:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 849 2024-09-18T00:07:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:07:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:07:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:19.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:19.310 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:07:19.314+0000 7f849c00d640 -1 reset not still connected to 0x5629c4a87d40 2024-09-18T00:07:19.310 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:07:19.314+0000 7f849c00d640 -1 reset not still connected to 0x5629cc435d40 2024-09-18T00:07:19.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 851 2024-09-18T00:07:19.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 850 2024-09-18T00:07:19.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:19.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:20.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:20.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:07:20.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:20.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:20.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:07:20.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:07:20.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:20.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:07:20.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:20.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:07:20.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:07:20.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 852 2024-09-18T00:07:20.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:21.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:21.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:07:21.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:21.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:21.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:07:21.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:07:21.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 854 2024-09-18T00:07:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:07:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:21.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:07:21.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:07:21.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 853 2024-09-18T00:07:21.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:21.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:22.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 855 2024-09-18T00:07:22.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 856 2024-09-18T00:07:22.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:22.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:22.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:22.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:07:22.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:22.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:22.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:07:22.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:07:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:07:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:07:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:07:22.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:22.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:07:22.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:22.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:22.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:07:22.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:07:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 857 2024-09-18T00:07:23.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 858 2024-09-18T00:07:23.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:23.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:07:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:07:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:07:24.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 859 2024-09-18T00:07:24.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 861 2024-09-18T00:07:24.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 860 2024-09-18T00:07:24.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:24.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:24.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:24.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:07:24.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:24.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:24.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:07:24.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:07:24.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:24.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:07:24.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:24.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:24.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:07:24.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:25.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 862 2024-09-18T00:07:25.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:07:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:07:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:07:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:07:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:07:25.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:07:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:07:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:07:26.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:07:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:07:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:07:26.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:07:26.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 863 2024-09-18T00:07:26.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 864 2024-09-18T00:07:26.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:26.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:27.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 865 2024-09-18T00:07:27.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 866 2024-09-18T00:07:27.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:27.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:27.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:07:27.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:27.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:27.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:07:27.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:07:27.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:07:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:07:27.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:07:27.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:27.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:07:27.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:27.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:27.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:07:27.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:07:28.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:28.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:07:28.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:28.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:28.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:07:28.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:07:28.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 868 2024-09-18T00:07:28.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 867 2024-09-18T00:07:28.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:28.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:29.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 870 2024-09-18T00:07:29.203 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:07:29.205+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d677b33930 2024-09-18T00:07:29.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:29.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:07:29.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:29.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:29.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:07:29.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:07:29.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 871 2024-09-18T00:07:29.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:29.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:07:29.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:29.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:29.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:07:29.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:29.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:29.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 869 2024-09-18T00:07:29.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:29.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:07:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:07:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:07:30.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:30.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:07:30.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:30.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:30.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:07:30.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:07:30.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 872 2024-09-18T00:07:30.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:31.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 874 2024-09-18T00:07:31.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:31.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:07:31.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:31.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:31.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:07:31.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:07:31.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:31.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:07:31.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:31.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:31.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:07:31.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:07:31.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 873 2024-09-18T00:07:31.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:31.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:32.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 876 2024-09-18T00:07:32.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 875 2024-09-18T00:07:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:32.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:32.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:32.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:07:32.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:32.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:32.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:07:32.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:07:32.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:32.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:07:32.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:32.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:32.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:07:32.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:07:32.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:07:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:07:32.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:07:33.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:33.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:07:33.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:33.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:33.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:07:33.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:07:33.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 877 2024-09-18T00:07:33.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:33.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 878 2024-09-18T00:07:33.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:34.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:34.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:07:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:07:34.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:07:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:07:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:07:34.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 880 2024-09-18T00:07:34.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 879 2024-09-18T00:07:34.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 881 2024-09-18T00:07:34.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:34.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:34.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:35.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:35.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:07:35.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:35.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:35.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:07:35.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:07:35.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:35.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:07:35.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:35.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:35.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:07:35.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:07:35.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 882 2024-09-18T00:07:35.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:36.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 883 2024-09-18T00:07:36.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 884 2024-09-18T00:07:36.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:36.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:36.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:07:36.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:36.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:36.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:07:36.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:07:36.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:36.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:36.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:07:36.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:36.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:36.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:07:36.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:07:37.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 885 2024-09-18T00:07:37.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 886 2024-09-18T00:07:37.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:37.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:07:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:07:37.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:07:37.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:37.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:07:37.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:37.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:37.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:07:37.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:07:37.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:37.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:07:37.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:37.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:37.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:07:37.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:07:38.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:38.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:07:38.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:38.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:38.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:07:38.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:07:38.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 888 2024-09-18T00:07:38.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:38.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 887 2024-09-18T00:07:38.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:39.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:39.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:07:39.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:39.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:39.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:07:39.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:39.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:07:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:07:39.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:07:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 890 2024-09-18T00:07:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 891 2024-09-18T00:07:39.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 889 2024-09-18T00:07:39.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:39.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:39.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:40.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:40.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:07:40.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:40.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:40.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:07:40.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:07:40.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:40.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:07:40.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:40.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:40.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:07:40.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:07:40.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 892 2024-09-18T00:07:40.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:41.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:41.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:07:41.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:41.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:41.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:07:41.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:07:41.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 893 2024-09-18T00:07:41.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:41.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:07:41.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:41.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:41.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:07:41.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:07:41.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 894 2024-09-18T00:07:41.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:41.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:42.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 895 2024-09-18T00:07:42.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 896 2024-09-18T00:07:42.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:42.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:07:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:07:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:07:42.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:42.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:07:42.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:42.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:42.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:07:42.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:07:42.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:42.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:07:42.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:42.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:42.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:07:42.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:07:43.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:43.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:07:43.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:43.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:43.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:07:43.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:07:43.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 897 2024-09-18T00:07:43.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 898 2024-09-18T00:07:43.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:43.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:44.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 900 2024-09-18T00:07:44.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 899 2024-09-18T00:07:44.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:44.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:44.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:07:44.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:44.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:44.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:07:44.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:44.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 901 2024-09-18T00:07:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:07:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:44.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:07:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:07:44.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:44.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:45.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:45.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:07:45.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:45.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:45.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:07:45.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:07:45.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:45.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:07:45.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:45.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:45.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:07:45.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:07:45.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 902 2024-09-18T00:07:45.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:46.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 903 2024-09-18T00:07:46.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 904 2024-09-18T00:07:46.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:46.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:07:46.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:46.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:46.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:07:46.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:07:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:07:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:07:46.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:07:46.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:46.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:47.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 905 2024-09-18T00:07:47.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 906 2024-09-18T00:07:47.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:47.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:47.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:47.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:07:47.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:47.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:47.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:07:47.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:07:47.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:47.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:07:47.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:47.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:47.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:07:47.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:07:47.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:47.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:07:47.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:47.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:47.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:07:47.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:07:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 907 2024-09-18T00:07:48.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 908 2024-09-18T00:07:48.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:48.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:48.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:07:48.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:48.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:48.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:07:48.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:07:49.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 911 2024-09-18T00:07:49.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 909 2024-09-18T00:07:49.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 910 2024-09-18T00:07:49.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:49.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:49.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:49.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:49.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:07:49.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:49.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:49.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:07:49.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:07:49.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:49.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:07:49.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:49.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:49.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:07:49.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:50.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 912 2024-09-18T00:07:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:50.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:50.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:07:50.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:50.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:50.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:07:50.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:07:50.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:50.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:07:50.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:50.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:50.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:07:50.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:07:51.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 913 2024-09-18T00:07:51.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 914 2024-09-18T00:07:51.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:51.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:07:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:07:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:07:51.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:51.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:07:51.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:51.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:51.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:07:51.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:07:51.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:51.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:52.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 915 2024-09-18T00:07:52.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:52.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 916 2024-09-18T00:07:52.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:52.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:52.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:07:52.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:52.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:52.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:07:52.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:07:52.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:52.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:07:52.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:52.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:52.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:07:52.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:07:53.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 917 2024-09-18T00:07:53.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:53.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:07:53.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:53.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:53.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:07:53.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:07:53.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 918 2024-09-18T00:07:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:53.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:54.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 920 2024-09-18T00:07:54.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 921 2024-09-18T00:07:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:54.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 919 2024-09-18T00:07:54.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:54.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:54.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:54.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:07:54.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:54.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:54.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:07:54.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:54.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:54.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:07:54.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:07:55.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:55.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:07:55.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:55.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:55.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:07:55.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:07:55.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 922 2024-09-18T00:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:07:55.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:07:55.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:56.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:56.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:07:56.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:56.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:56.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:07:56.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:07:56.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:56.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:07:56.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:56.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:56.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:07:56.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:07:56.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 924 2024-09-18T00:07:56.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 923 2024-09-18T00:07:56.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:56.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:57.025 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:07:57.024+0000 7fac32fac640 -1 reset not still connected to 0x55a841573ba0 2024-09-18T00:07:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:07:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:07:57.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 925 2024-09-18T00:07:57.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:07:57.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 926 2024-09-18T00:07:57.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:57.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:57.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:57.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:07:57.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:57.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:57.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:07:57.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:07:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:07:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:07:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:07:58.012 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:07:58.013+0000 7fd4c158b640 -1 reset not still connected to 0x56435ba7fa00 2024-09-18T00:07:58.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 928 2024-09-18T00:07:58.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:58.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:07:58.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:58.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:58.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:07:58.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:07:58.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 927 2024-09-18T00:07:58.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:58.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:59.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:59.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:07:59.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:07:59.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:07:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:07:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:07:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:07:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:07:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:07:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:07:59.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 930 2024-09-18T00:07:59.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 931 2024-09-18T00:07:59.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 929 2024-09-18T00:07:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:59.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:07:59.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:08:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:08:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:08:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:08:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:08:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:08:00.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 932 2024-09-18T00:08:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:01.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:01.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:08:01.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:01.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:01.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:08:01.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:08:01.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:01.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:08:01.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:01.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:01.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:08:01.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:08:01.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 933 2024-09-18T00:08:01.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 934 2024-09-18T00:08:01.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:01.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:02.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 935 2024-09-18T00:08:02.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 936 2024-09-18T00:08:02.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:02.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:02.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:02.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:08:02.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:02.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:02.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:08:02.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:08:02.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:02.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:08:02.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:02.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:02.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:08:02.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:08:02.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:02.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:08:02.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:02.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:02.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:08:02.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:08:03.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:03.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:08:03.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:03.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:03.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:08:03.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:08:03.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 937 2024-09-18T00:08:03.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 938 2024-09-18T00:08:03.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:03.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:04.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 941 2024-09-18T00:08:04.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:04.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 939 2024-09-18T00:08:04.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 940 2024-09-18T00:08:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:04.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:08:04.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:04.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:04.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:08:04.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:08:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:04.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:08:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:08:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:08:05.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:05.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:08:05.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:05.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:05.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:08:05.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:08:05.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:05.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:08:05.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:05.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:05.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:08:05.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:08:05.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 942 2024-09-18T00:08:05.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:06.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:06.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:08:06.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:06.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:06.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:08:06.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:08:06.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 944 2024-09-18T00:08:06.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:06.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:08:06.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:06.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:06.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:08:06.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:08:06.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 943 2024-09-18T00:08:06.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:06.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:07.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 946 2024-09-18T00:08:07.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 945 2024-09-18T00:08:07.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:07.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:07.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:07.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:08:07.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:07.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:07.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:08:07.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:08:07.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:07.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:08:07.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:07.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:07.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:08:07.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:08:07.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:08:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:08:07.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:08:08.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 947 2024-09-18T00:08:08.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:08.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:08:08.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:08.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:08.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:08:08.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:08:08.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 948 2024-09-18T00:08:08.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:08.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:09.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 951 2024-09-18T00:08:09.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 949 2024-09-18T00:08:09.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 950 2024-09-18T00:08:09.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:09.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:08:09.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:09.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:09.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:08:09.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:08:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:08:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:08:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:08:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:09.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:09.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:09.954 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:08:09.956+0000 7fac32fac640 -1 reset not still connected to 0x55a83a519e10 2024-09-18T00:08:10.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:10.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:08:10.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:10.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:10.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:08:10.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:08:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:08:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:08:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:08:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 952 2024-09-18T00:08:10.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:08:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:08:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:08:11.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 954 2024-09-18T00:08:11.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:11.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:08:11.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:11.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:11.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:08:11.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:08:11.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 953 2024-09-18T00:08:11.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:11.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 956 2024-09-18T00:08:12.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:12.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 955 2024-09-18T00:08:12.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:12.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:12.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:12.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:08:12.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:08:12.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:12.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:12.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:12.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:08:12.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:12.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:08:12.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:08:12.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:08:12.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:12.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:08:12.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:12.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:12.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:08:12.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:08:13.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:13.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:08:13.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:13.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:13.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:08:13.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:08:13.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 957 2024-09-18T00:08:13.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 958 2024-09-18T00:08:13.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:13.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:14.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 961 2024-09-18T00:08:14.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 960 2024-09-18T00:08:14.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 959 2024-09-18T00:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:14.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:14.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:08:14.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:08:14.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:14.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:08:14.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:14.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:14.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:08:14.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:08:14.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:14.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:14.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:15.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:08:15.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 52397 2024-09-18T00:08:15.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:08:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:08:15.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:08:15.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 962 2024-09-18T00:08:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:16.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 963 2024-09-18T00:08:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 964 2024-09-18T00:08:16.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:16.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:16.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:08:16.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 52535 2024-09-18T00:08:16.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:16.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:08:16.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:16.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:16.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:08:16.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:08:17.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 965 2024-09-18T00:08:17.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:08:17.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 52668 2024-09-18T00:08:17.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:08:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:08:17.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:08:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:08:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:08:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:08:18.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 966 2024-09-18T00:08:18.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:18.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:08:18.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 52826 2024-09-18T00:08:19.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 967 2024-09-18T00:08:19.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 968 2024-09-18T00:08:19.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:19.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:08:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 53065 2024-09-18T00:08:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:08:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:08:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:08:20.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:20.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:08:20.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:20.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:20.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:08:20.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:08:21.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 969 2024-09-18T00:08:21.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:08:21.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 53239 2024-09-18T00:08:21.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:22.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 970 2024-09-18T00:08:22.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:08:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 53433 2024-09-18T00:08:22.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:22.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:08:22.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:22.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:22.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:08:22.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:08:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:08:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:08:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:08:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:08:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:08:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:08:24.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 971 2024-09-18T00:08:24.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:08:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 53662 2024-09-18T00:08:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 972 2024-09-18T00:08:26.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:08:27.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:08:27.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 53960 2024-09-18T00:08:29.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:08:29.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:08:29.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 52264 2024-09-18T00:10:47.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:10:47.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:10:47.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:10:47.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 52398 2024-09-18T00:10:48.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:10:48.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:10:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:10:48.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 52536 2024-09-18T00:10:49.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:10:49.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:10:49.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:10:49.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 52669 2024-09-18T00:10:50.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:10:50.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:10:50.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:10:50.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 52827 2024-09-18T00:10:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:10:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:10:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:10:51.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 53066 2024-09-18T00:10:52.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:10:52.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:10:52.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:10:52.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 53240 2024-09-18T00:10:54.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:10:54.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:10:54.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:10:54.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 53434 2024-09-18T00:10:56.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:10:56.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:10:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:10:56.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 53663 2024-09-18T00:10:58.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:10:58.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:10:58.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:10:58.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 53961 2024-09-18T00:11:00.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:11:00.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:11:00.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:11:00.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:11:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:11:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:11:00.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 1 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd0 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary1 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary1 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary1 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 2 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 3 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd1 2024-09-18T00:11:00.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary2 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary2 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary2 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd2 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary3 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary3 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary3 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 4 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:11:00.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd3 2024-09-18T00:11:00.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 5 2024-09-18T00:11:00.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary4 2024-09-18T00:11:00.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary4 2024-09-18T00:11:00.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary4 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 6 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd4 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd5 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary5 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary5 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary5 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:11:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary6 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary6 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary6 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 7 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd6 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary7 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary7 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:11:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary7 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 60007 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 9 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd8 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary9 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary9 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary9 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 10 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd9 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 8 2024-09-18T00:11:00.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary10 2024-09-18T00:11:00.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd7 2024-09-18T00:11:00.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary10 2024-09-18T00:11:00.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:11:00.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary10 2024-09-18T00:11:00.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary8 2024-09-18T00:11:00.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary8 2024-09-18T00:11:00.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:11:00.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary8 2024-09-18T00:11:00.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:11:00.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:11:00.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:11:00.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:11:00.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:11:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:11:00.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:11:00.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:11:00.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:11:00.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:11:00.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:11:00.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:11:00.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:11:00.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:11:00.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:11:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:11:00.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:11:00.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:11:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:11:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:11:59.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=794c634cd3fe74e62ec69cadae508d46 2024-09-18T00:11:59.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary6 2024-09-18T00:12:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=94c52a39d7d75b807f0b1d7bb64168f8 2024-09-18T00:12:00.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary3 2024-09-18T00:12:00.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=6452d440b68c7dde7b2ec5b8efcb9544 2024-09-18T00:12:00.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary5 2024-09-18T00:12:01.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=d3e29ccb569c89f02743be8bf4d41e29 2024-09-18T00:12:01.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary2 2024-09-18T00:12:01.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=c548cdf6a3f9777a69febac611824e2f 2024-09-18T00:12:01.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary1 2024-09-18T00:12:01.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary6 2024-09-18T00:12:01.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:01.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:01.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:12:01.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary6 2024-09-18T00:12:01.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary3 2024-09-18T00:12:01.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:01.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:01.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:12:01.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary3 2024-09-18T00:12:02.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=cac5b70057e374f548dc6ead679a7b2a 2024-09-18T00:12:02.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary7 2024-09-18T00:12:02.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary5 2024-09-18T00:12:02.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:02.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:02.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:12:02.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary5 2024-09-18T00:12:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary1 2024-09-18T00:12:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:12:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary1 2024-09-18T00:12:02.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary2 2024-09-18T00:12:02.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:02.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:02.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:12:02.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary2 2024-09-18T00:12:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=890c56486a23d7d06aa79071d2970819 2024-09-18T00:12:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary8 2024-09-18T00:12:03.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary7 2024-09-18T00:12:03.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:03.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:12:03.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary7 2024-09-18T00:12:03.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:12:03.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:12:03.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+unknown 2024-09-18T00:12:03.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:03.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:03.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:12:03.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:03.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:03.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:03.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:03.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:03.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+unknown 2024-09-18T00:12:03.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:03.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:03.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:12:03.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:03.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:03.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:03.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:03.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:03.922 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:12:03.915+0000 7fd4c158b640 -1 reset not still connected to 0x564358707450 2024-09-18T00:12:03.922 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:12:03.915+0000 7fd4c158b640 -1 reset not still connected to 0x5643587075f0 2024-09-18T00:12:03.922 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:12:03.915+0000 7fd4c158b640 -1 reset not still connected to 0x564358707ba0 2024-09-18T00:12:03.922 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:12:03.915+0000 7fd4c158b640 -1 reset not still connected to 0x564358707d40 2024-09-18T00:12:03.922 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:12:03.915+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a340 2024-09-18T00:12:03.923 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:12:03.915+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T00:12:03.923 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:12:03.915+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T00:12:03.923 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:12:03.915+0000 7fd4c158b640 -1 reset not still connected to 0x5643595bdc70 2024-09-18T00:12:03.923 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:12:03.915+0000 7fd4c158b640 -1 reset not still connected to 0x56435b71c270 2024-09-18T00:12:03.923 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:12:03.915+0000 7fd4c158b640 -1 reset not still connected to 0x56435ba7fa00 2024-09-18T00:12:03.923 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:12:03.915+0000 7fd4c158b640 -1 reset not still connected to 0x56435da4ea90 2024-09-18T00:12:04.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary8 2024-09-18T00:12:04.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:04.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:04.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:12:04.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary8 2024-09-18T00:12:04.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=1d761a4f264508c80ad0f497b0c1c53b 2024-09-18T00:12:04.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary10 2024-09-18T00:12:04.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:12:04.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:12:04.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:12:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+unknown 2024-09-18T00:12:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:12:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:04.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+unknown 2024-09-18T00:12:04.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:04.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:04.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:12:04.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:04.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:04.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:04.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:04.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:04.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T00:12:04.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:04.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:04.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:12:04.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:04.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:04.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:04.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:04.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T00:12:04.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:04.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:04.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:12:04.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:04.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:04.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:04.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:04.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:04.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:12:04.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:04.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:12:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+unknown 2024-09-18T00:12:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:04.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:12:04.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:04.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:04.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:04.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:04.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:04.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=9c52270816e3fc83308a4a7a9f0b2315 2024-09-18T00:12:04.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary4 2024-09-18T00:12:04.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:12:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 49280c03-8d1c-4c44-b8cc-dc519e871b86 2024-09-18T00:12:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04' 2024-09-18T00:12:04.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:04.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:12:04.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: e7927f4e-32c4-4eaa-be90-8922d3a6eb2f 2024-09-18T00:12:04.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:04.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:04.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:04.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:04.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:04.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:04.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:04.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:04.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04' 2024-09-18T00:12:04.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:04.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:04.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:04.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:04.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:04.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:04.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:04.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:04.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:04.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:04.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:04.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:04.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:04.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:04.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:04.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:04.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:04.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:04.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:04.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+unknown '2024-09-18 00:12:04' 2024-09-18T00:12:04.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:04.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:12:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:04' 2024-09-18T00:12:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+unknown '2024-09-18 00:12:04' 2024-09-18T00:12:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:04.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:04.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:12:04.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:04.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:04' 2024-09-18T00:12:04.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:04.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:04.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:04.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:04.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:04.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=24709b11fb2630a9bbf4a7e0fa08f708 2024-09-18T00:12:04.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary9 2024-09-18T00:12:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2623488.0,"last_snapshot_bytes":5120000,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618091,"remote_snapshot_timestamp":1726618091,"replay_state":"idle"}2024-09-18 00:11:39image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2676736.0,"last_snapshot_bytes":5218304,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618105,"remote_snapshot_timestamp":1726618105,"replay_state":"idle"}2024-09-18 00:11:39image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618092,"remote_snapshot_timestamp":1726618092,"replay_state":"idle"}2024-09-18 00:11:39image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:04image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618094,"remote_snapshot_timestamp":1726618094,"replay_state":"idle"}2024-09-18 00:11:39image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":950272.0,"last_snapshot_bytes":1703936,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618095,"remote_snapshot_timestamp":1726618095,"replay_state":"idle"}2024-09-18 00:11:39image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:04image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":178176.0,"last_snapshot_bytes":135168,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618098,"remote_snapshot_timestamp":1726618098,"replay_state":"idle"}2024-09-18 00:11:39image-primary82846127d-142c-45b9-b096-a07f221235bdup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2584576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618101,"remote_snapshot_timestamp":1726618101,"replay_state":"idle"}2024-09-18 00:11:39image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2596864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618103,"remote_snapshot_timestamp":1726618103,"replay_state":"idle"}2024-09-18 00:11:39' 2024-09-18T00:12:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:12:04.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:04.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:12:04.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:04.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:04.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:04.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary6 up+unknown 2024-09-18T00:12:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:12:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:04.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 00:12:04image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2676736.0,"last_snapshot_bytes":5218304,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618105,"remote_snapshot_timestamp":1726618105,"replay_state":"idle"}2024-09-18 00:11:39image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 00:12:04image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:04image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618094,"remote_snapshot_timestamp":1726618094,"replay_state":"idle"}2024-09-18 00:11:39image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 00:12:04image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:04image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":178176.0,"last_snapshot_bytes":135168,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618098,"remote_snapshot_timestamp":1726618098,"replay_state":"idle"}2024-09-18 00:11:39image-primary82846127d-142c-45b9-b096-a07f221235bdup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2584576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618101,"remote_snapshot_timestamp":1726618101,"replay_state":"idle"}2024-09-18 00:11:39image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2596864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618103,"remote_snapshot_timestamp":1726618103,"replay_state":"idle"}2024-09-18 00:11:39' 2024-09-18T00:12:04.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:12:04.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:04.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:12:04.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:04.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:04.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:04.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:04.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:04.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:04.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary3 up+unknown 2024-09-18T00:12:04.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:04.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:04.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:12:04.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:04.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:04.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:04.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:04.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:05.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary10 2024-09-18T00:12:05.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:05.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:12:05.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary10 2024-09-18T00:12:05.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary4 2024-09-18T00:12:05.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:05.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:12:05.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary4 2024-09-18T00:12:05.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:12:05.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:12:05.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:05.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:12:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:12:05.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:12:05.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:12:05.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:05.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:12:05.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:05.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:05.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:05.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:05.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:12:05.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+unknown 2024-09-18T00:12:05.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:05.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:12:05.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:05.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:05.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:05.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:05.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+unknown 2024-09-18T00:12:05.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:05.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:12:05.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:05.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:05.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:05.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:05.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:12:05.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:05.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:12:05.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:05.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:05.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:05.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:05.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f00bdbdc-f2bd-4210-80a6-7c7769eada40 2024-09-18T00:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04' 2024-09-18T00:12:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:05.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:05.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:05.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:05.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:05.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:05.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:05.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:05.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:05.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:05.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:05.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:05.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:05.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+unknown '2024-09-18 00:12:04' 2024-09-18T00:12:05.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:05.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:12:05.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:04' 2024-09-18T00:12:05.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:05.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:05.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:05.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:12:05.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: fac94502-f15a-4e8a-b812-20cd1df947cf 2024-09-18T00:12:05.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:05.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:05.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04' 2024-09-18T00:12:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:05.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:05.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:05.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:05.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:05.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:05.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:05.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:05.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:05.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:05.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:05.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:05.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+unknown '2024-09-18 00:12:04' 2024-09-18T00:12:05.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:05.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:12:05.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:04' 2024-09-18T00:12:05.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:05.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:05.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:05.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary9 2024-09-18T00:12:05.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:05.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:12:05.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary9 2024-09-18T00:12:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:12:04image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2676736.0,"last_snapshot_bytes":5218304,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618105,"remote_snapshot_timestamp":1726618105,"replay_state":"idle"}2024-09-18 00:11:39image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:12:04image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:04image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618094,"remote_snapshot_timestamp":1726618094,"replay_state":"idle"}2024-09-18 00:11:39image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:12:04image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:04image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":178176.0,"last_snapshot_bytes":135168,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618098,"remote_snapshot_timestamp":1726618098,"replay_state":"idle"}2024-09-18 00:11:39image-primary82846127d-142c-45b9-b096-a07f221235bdup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2584576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618101,"remote_snapshot_timestamp":1726618101,"replay_state":"idle"}2024-09-18 00:11:39image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2596864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618103,"remote_snapshot_timestamp":1726618103,"replay_state":"idle"}2024-09-18 00:11:39' 2024-09-18T00:12:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:12:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:12:05.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:05.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:05.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:05.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:05.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:05.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:05.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary2 up+unknown 2024-09-18T00:12:05.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:05.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:12:05.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:05.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:05.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:05.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:05.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:12:04image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2676736.0,"last_snapshot_bytes":5218304,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618105,"remote_snapshot_timestamp":1726618105,"replay_state":"idle"}2024-09-18 00:11:39image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:12:04image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:04image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618094,"remote_snapshot_timestamp":1726618094,"replay_state":"idle"}2024-09-18 00:11:39image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:12:04image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:04image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":178176.0,"last_snapshot_bytes":135168,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618098,"remote_snapshot_timestamp":1726618098,"replay_state":"idle"}2024-09-18 00:11:39image-primary82846127d-142c-45b9-b096-a07f221235bdup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2584576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618101,"remote_snapshot_timestamp":1726618101,"replay_state":"idle"}2024-09-18 00:11:39image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2596864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618103,"remote_snapshot_timestamp":1726618103,"replay_state":"idle"}2024-09-18 00:11:39' 2024-09-18T00:12:05.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:05.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:12:05.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:05.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:12:05.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:05.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:05.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:05.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:05.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:05.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:05.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary5 up+unknown 2024-09-18T00:12:05.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:05.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:12:05.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:05.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:05.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:05.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:05.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:12:05.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: e748bb2a-a5aa-46c1-b88f-a4adee5d24c6 2024-09-18T00:12:05.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:05.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:05.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:05.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:05.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:05.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:05.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:05.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:05.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04' 2024-09-18T00:12:05.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:05.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:05.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:05.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:05.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:05.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:05.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:05.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:05.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:05.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:05.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:05.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:05.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:05.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+unknown '2024-09-18 00:12:04' 2024-09-18T00:12:05.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:12:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:04' 2024-09-18T00:12:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary6 up+unknown '' '' 2024-09-18T00:12:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:12:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:05.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:05.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary6 2024-09-18T00:12:05.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:12:04image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2676736.0,"last_snapshot_bytes":5218304,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618105,"remote_snapshot_timestamp":1726618105,"replay_state":"idle"}2024-09-18 00:11:39image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:12:04image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:04image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618094,"remote_snapshot_timestamp":1726618094,"replay_state":"idle"}2024-09-18 00:11:39image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:12:04image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:04image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":178176.0,"last_snapshot_bytes":135168,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618098,"remote_snapshot_timestamp":1726618098,"replay_state":"idle"}2024-09-18 00:11:39image-primary82846127d-142c-45b9-b096-a07f221235bdup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2584576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618101,"remote_snapshot_timestamp":1726618101,"replay_state":"idle"}2024-09-18 00:11:39image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:12:04cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2596864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618103,"remote_snapshot_timestamp":1726618103,"replay_state":"idle"}2024-09-18 00:11:39' 2024-09-18T00:12:05.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:05.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:12:05.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:05.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:12:05.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:05.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:05.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:05.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:05.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:05.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary1 up+unknown 2024-09-18T00:12:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:12:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:05.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:12:05.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: e7927f4e-32c4-4eaa-be90-8922d3a6eb2f 2024-09-18T00:12:05.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:12:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04' 2024-09-18T00:12:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:05.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:05.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:05.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:05.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:05.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:05.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:05.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:05.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:05.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:05.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:05.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:05.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:05.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary6 up+unknown '2024-09-18 00:12:04' 2024-09-18T00:12:05.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:05.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:12:05.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:04' 2024-09-18T00:12:05.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:05.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:05.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:05.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary3 up+unknown '' '' 2024-09-18T00:12:05.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:05.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:05.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:12:05.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:05.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:05.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:05.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:05.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:05.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary3 2024-09-18T00:12:06.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:12:06.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 49280c03-8d1c-4c44-b8cc-dc519e871b86 2024-09-18T00:12:06.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:06.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:06.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:06.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:06.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:06.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:12:06.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:06.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:06.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04' 2024-09-18T00:12:06.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:06.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:06.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:06.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:06.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:06.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:06.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:06.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:06.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:06.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING7342614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2676736.0,"last_snapshot_bytes":5218304,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618105,"remote_snapshot_timestamp":1726618105,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618094,"remote_snapshot_timestamp":1726618094,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:05cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:05cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2596864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618103,"remote_snapshot_timestamp":1726618103,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04' 2024-09-18T00:12:06.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:06.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:12:06.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:06.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:06.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:06.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:06.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary3 up+unknown '2024-09-18 00:12:04' 2024-09-18T00:12:06.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:06.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:06.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:12:06.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:06.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:04' 2024-09-18T00:12:06.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:06.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:06.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:06.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:06.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:12:06.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:06.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:06.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:06.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:06.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:06.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:06.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary6 2024-09-18T00:12:06.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:06.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:06.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:12:06.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:12:06.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary6 2024-09-18T00:12:06.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING7342614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2676736.0,"last_snapshot_bytes":5218304,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618105,"remote_snapshot_timestamp":1726618105,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618094,"remote_snapshot_timestamp":1726618094,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:05cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:05cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2596864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618103,"remote_snapshot_timestamp":1726618103,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04' 2024-09-18T00:12:06.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:06.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:12:06.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:06.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:12:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:06.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:06.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:06.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary3 2024-09-18T00:12:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:12:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:12:06.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary3 2024-09-18T00:12:06.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:12:06.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+unknown 2024-09-18T00:12:06.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:06.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:06.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:12:06.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:06.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:06.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:06.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:06.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:12:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:12:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:06.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:06.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:06.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:06.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:06.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:12:06.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:12:06.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:06.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:06.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:12:06.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:06.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:06.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:06.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:06.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:06.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:12:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:12:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 2846127d-142c-45b9-b096-a07f221235bd 2024-09-18T00:12:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:12:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:12:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:05' 2024-09-18T00:12:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:12:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:12:06.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:12:06.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 5cbf0aba-1bfb-4812-b772-7406d18bca14 2024-09-18T00:12:06.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:12:06.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:12:06.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:06.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:06.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:06.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:06.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:06.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:06.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:05' 2024-09-18T00:12:06.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:06.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:12:06.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:06.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:12:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary2 up+unknown '' '' 2024-09-18T00:12:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:12:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:06.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary2 2024-09-18T00:12:06.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary5 up+unknown '' '' 2024-09-18T00:12:06.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:06.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:06.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:12:06.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:06.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:06.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:06.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:06.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:06.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary5 2024-09-18T00:12:06.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:12:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f00bdbdc-f2bd-4210-80a6-7c7769eada40 2024-09-18T00:12:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:12:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04' 2024-09-18T00:12:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:06.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:06.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:06.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:12:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:06.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:06.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary2 up+unknown '2024-09-18 00:12:04' 2024-09-18T00:12:06.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:06.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:06.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:12:06.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:06.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:04' 2024-09-18T00:12:06.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:06.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:06.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:06.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:12:06.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: fac94502-f15a-4e8a-b812-20cd1df947cf 2024-09-18T00:12:06.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:12:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04' 2024-09-18T00:12:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:06.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:06.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:06.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:12:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:06.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:06.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:06.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:06.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:06.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary5 up+unknown '2024-09-18 00:12:04' 2024-09-18T00:12:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:12:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:04' 2024-09-18T00:12:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:06.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:06.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING7342614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2676736.0,"last_snapshot_bytes":5218304,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618105,"remote_snapshot_timestamp":1726618105,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618094,"remote_snapshot_timestamp":1726618094,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:05cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:05cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2596864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618103,"remote_snapshot_timestamp":1726618103,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04' 2024-09-18T00:12:06.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:06.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:12:06.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:06.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:12:06.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:06.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:06.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:06.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:06.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:06.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:06.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary2 2024-09-18T00:12:06.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:06.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:06.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:12:06.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:12:06.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary2 2024-09-18T00:12:06.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary1 up+unknown '' '' 2024-09-18T00:12:06.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:06.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:06.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:12:06.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:06.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:06.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:06.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:06.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:06.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary1 2024-09-18T00:12:06.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:12:06.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:12:06.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:12:06.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:12:06.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:12:06.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:12:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:12:06.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:12:06.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:12:06.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:12:06.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: e748bb2a-a5aa-46c1-b88f-a4adee5d24c6 2024-09-18T00:12:06.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:06.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:06.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:06.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:06.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:06.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:12:06.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:06.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:06.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04' 2024-09-18T00:12:06.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:06.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:06.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:06.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:12:06.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:06.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:06.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:06.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:07.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:07.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:07.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:07.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:07.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:07.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary1 up+unknown '2024-09-18 00:12:04' 2024-09-18T00:12:07.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:07.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:07.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:12:07.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:07.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:04' 2024-09-18T00:12:07.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:07.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:07.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:07.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING7342614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:06cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618094,"remote_snapshot_timestamp":1726618094,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:05cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:05cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2596864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618103,"remote_snapshot_timestamp":1726618103,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04' 2024-09-18T00:12:07.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:07.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:12:07.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:07.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:12:07.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:07.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:07.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:07.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:07.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:07.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:07.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary5 2024-09-18T00:12:07.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:07.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:07.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:12:07.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:12:07.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary5 2024-09-18T00:12:07.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd0 2024-09-18T00:12:07.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:12:07.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:12:07.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd1 2024-09-18T00:12:07.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:12:07.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:12:07.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd2 2024-09-18T00:12:07.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:12:07.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:12:07.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:06cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":958464.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618094,"remote_snapshot_timestamp":1726618094,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:04image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:05cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:12:05cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2596864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618103,"remote_snapshot_timestamp":1726618103,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:11:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:12:04' 2024-09-18T00:12:07.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:07.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:12:07.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:04' 2024-09-18T00:12:07.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:12:07.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:07.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:07.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:07.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary1 2024-09-18T00:12:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:12:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:12:07.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary1 2024-09-18T00:12:07.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T00:12:07.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:07.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:07.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:12:07.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:07.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:07.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:07.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:07.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:07.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:12:07.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:12:07.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:12:07.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+unknown 2024-09-18T00:12:07.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:07.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:07.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:12:07.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:07.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:07.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:07.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:07.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:07.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+unknown 2024-09-18T00:12:07.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:07.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:07.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:12:07.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:07.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:07.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:07.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:07.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:12:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 02b36501-4523-474d-b4d9-f59723d6b12e 2024-09-18T00:12:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:12:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:12:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:07.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:06' 2024-09-18T00:12:07.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:07.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:12:07.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:07.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:12:07.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:12:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:12:07.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:12:07.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:12:07.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:12:07.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:12:08.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd3 2024-09-18T00:12:08.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:12:08.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:12:08.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd4 2024-09-18T00:12:08.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:12:08.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:12:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:12:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:12:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:12:08.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T00:12:08.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:08.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:08.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:12:08.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:08.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:08.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:08.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:08.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:08.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:12:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:12:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:12:08.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:08.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:08.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:08.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:08.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:08.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:12:08.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:12:08.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:08.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:08.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:12:08.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:08.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:08.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:08.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:08.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:08.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:12:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:12:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: bcd32a3a-52fc-4137-aa0a-509317d6523f 2024-09-18T00:12:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:12:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:12:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:08.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:08.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:08.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:08.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:07' 2024-09-18T00:12:08.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:08.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:12:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3 2024-09-18T00:12:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:12:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:12:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:07' 2024-09-18T00:12:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:12:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:12:08.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:12:08.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:08.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 5cbf0aba-1bfb-4812-b772-7406d18bca14 2024-09-18T00:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:08.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:08.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:08.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:08.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:05' 2024-09-18T00:12:08.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:08.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:12:08.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:08.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:12:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:12:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 2846127d-142c-45b9-b096-a07f221235bd 2024-09-18T00:12:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:12:08.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:12:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:04 2024-09-18T00:12:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:12:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:05' 2024-09-18T00:12:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:08.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:12:08.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:08.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:12:09.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T00:12:09.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:09.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:09.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:12:09.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:09.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:09.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:09.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:09.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:09.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:12:09.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:12:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 02b36501-4523-474d-b4d9-f59723d6b12e 2024-09-18T00:12:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09 2024-09-18T00:12:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09' 2024-09-18T00:12:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:09.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:09.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:09.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:09.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:09.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:09.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:09.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:09.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:09.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+unknown '2024-09-18 00:12:09' 2024-09-18T00:12:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:12:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:09' 2024-09-18T00:12:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:10.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09' 2024-09-18T00:12:10.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:10.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:12:10.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:10.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:12:10.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:10.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:10.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:10.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary10 up+unknown 2024-09-18T00:12:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:12:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:10.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:10.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T00:12:10.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:10.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:10.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:12:10.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:10.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:10.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:10.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:12:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:12:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:10.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:10.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:10.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:12:10.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:10.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:12:10.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:12:10.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: bcd32a3a-52fc-4137-aa0a-509317d6523f 2024-09-18T00:12:10.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:10.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:10.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:10.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09 2024-09-18T00:12:10.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:10.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:10.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:10.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:10.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09' 2024-09-18T00:12:10.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:10.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:10.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:10.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:10.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:10.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:10.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:10.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:10.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:10.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:10.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:10.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+unknown '2024-09-18 00:12:09' 2024-09-18T00:12:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:12:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:09' 2024-09-18T00:12:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:10.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:12:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3 2024-09-18T00:12:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09 2024-09-18T00:12:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:10.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:10.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:10.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09' 2024-09-18T00:12:10.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:10.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:10.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:10.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:10.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:10.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:10.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:10.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:10.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:10.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:10.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+unknown '2024-09-18 00:12:09' 2024-09-18T00:12:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:12:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:09' 2024-09-18T00:12:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:11.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09' 2024-09-18T00:12:11.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:11.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:12:11.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:11.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:12:11.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:11.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:11.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:11.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:11.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:11.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:11.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary9 up+unknown 2024-09-18T00:12:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:12:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:11.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:11.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09' 2024-09-18T00:12:11.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:11.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:12:11.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:11.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:12:11.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:11.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:11.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:11.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:11.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary4 up+unknown 2024-09-18T00:12:11.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:11.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:11.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:12:11.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:11.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:11.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:11.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:11.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:11.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary10 up+unknown '' '' 2024-09-18T00:12:11.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:11.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:11.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:12:11.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:11.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:11.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:11.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:11.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:11.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary10 2024-09-18T00:12:11.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:12:11.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 02b36501-4523-474d-b4d9-f59723d6b12e 2024-09-18T00:12:11.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:11.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:11.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:11.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09 2024-09-18T00:12:11.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:11.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:12:11.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:11.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:11.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09' 2024-09-18T00:12:11.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:11.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:11.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:11.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:11.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:11.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:11.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:11.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:11.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary10 up+unknown '2024-09-18 00:12:09' 2024-09-18T00:12:11.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:12:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:09' 2024-09-18T00:12:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:11.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}2024-09-18 00:12:09image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}2024-09-18 00:12:09image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09' 2024-09-18T00:12:11.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:11.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:12:11.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:11.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:12:11.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:11.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:11.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:11.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:11.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:11.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:11.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary10 2024-09-18T00:12:11.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:11.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:11.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:12:11.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:12:11.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary10 2024-09-18T00:12:11.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:12:11.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:12:11.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:12:12.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary9 up+unknown '' '' 2024-09-18T00:12:12.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:12.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:12.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:12:12.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:12.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:12.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:12.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:12.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:12.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary9 2024-09-18T00:12:12.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary4 up+unknown '' '' 2024-09-18T00:12:12.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:12.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:12.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:12:12.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:12.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:12.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:12.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:12.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:12.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary4 2024-09-18T00:12:12.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd5 2024-09-18T00:12:12.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:12:12.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:12:12.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:12:12.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: bcd32a3a-52fc-4137-aa0a-509317d6523f 2024-09-18T00:12:12.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:12.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:12.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:12.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09 2024-09-18T00:12:12.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:12.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:12:12.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:12.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:12.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09' 2024-09-18T00:12:12.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:12.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:12.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:12.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:12.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:12.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:12.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:12.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:12.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:12.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:12.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:12.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:12.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary9 up+unknown '2024-09-18 00:12:09' 2024-09-18T00:12:12.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:12.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:12.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:12:12.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:12.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:09' 2024-09-18T00:12:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:12.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:12.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:12:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3 2024-09-18T00:12:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09 2024-09-18T00:12:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:12:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09' 2024-09-18T00:12:12.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:12.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:12.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:12.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:12.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:12.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:12.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:12.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:12.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:12.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:12.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:12.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:12.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:12.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary4 up+unknown '2024-09-18 00:12:09' 2024-09-18T00:12:12.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:12.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:12.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:12:12.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:12.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:09' 2024-09-18T00:12:12.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:12.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:12.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:12.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}2024-09-18 00:12:09image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}2024-09-18 00:12:09image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09' 2024-09-18T00:12:12.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:12.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:12:12.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}2024-09-18 00:12:09image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}2024-09-18 00:12:09image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:09' 2024-09-18T00:12:12.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:12.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:12:12.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:12.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:12:12.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:12.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:12:12.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:12:12.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:12.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:12.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:12:12.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:12.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:12.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:12.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:12.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:12.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:12:12.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:12.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:12.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:12.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:12.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:12.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:12.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary4 2024-09-18T00:12:12.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:12.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:12.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:12:12.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:12:12.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary4 2024-09-18T00:12:12.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:12.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:12.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:12.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:12.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:12.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:12.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary9 2024-09-18T00:12:12.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:12.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:12.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:12:12.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:12:12.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary9 2024-09-18T00:12:12.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:12:12.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:12.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:12.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:12:12.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:12.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:12.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:12:12.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:12:12.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 5cbf0aba-1bfb-4812-b772-7406d18bca14 2024-09-18T00:12:12.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:12.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:12.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:12.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09 2024-09-18T00:12:12.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:12.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:12.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:12.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:12.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09' 2024-09-18T00:12:12.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:12.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:12.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:12.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:12.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:12.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:12.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:12.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:12.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:12:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 2846127d-142c-45b9-b096-a07f221235bd 2024-09-18T00:12:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09 2024-09-18T00:12:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:12:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:09' 2024-09-18T00:12:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:12.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:12.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:12.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:12.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:12.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:12.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:12.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+unknown '2024-09-18 00:12:09' 2024-09-18T00:12:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:12:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:09' 2024-09-18T00:12:12.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:12.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:12.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:12.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:12.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:12.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:12.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:12.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:12.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:12.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:12.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+unknown '2024-09-18 00:12:09' 2024-09-18T00:12:12.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:12:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:12:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:09' 2024-09-18T00:12:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:12.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:13.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09' 2024-09-18T00:12:13.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:13.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:12:13.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:12:09image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:12:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:12:09' 2024-09-18T00:12:13.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:13.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:12:13.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:13.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:12:13.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:09' 2024-09-18T00:12:13.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:13.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:13.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:13.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:12:13.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:13.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:13.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary8 up+unknown 2024-09-18T00:12:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:12:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:13.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:13.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:13.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:13.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:13.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:13.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:13.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary7 up+unknown 2024-09-18T00:12:13.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:13.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:13.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:12:13.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:13.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:13.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:13.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:12:13.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:12:13.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:12:13.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:12:13.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:12:13.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:12:13.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:12:13.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:12:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary8 up+unknown '' '' 2024-09-18T00:12:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:12:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:14.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:14.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:14.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary8 2024-09-18T00:12:14.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary7 up+unknown '' '' 2024-09-18T00:12:14.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:14.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:14.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:12:14.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:14.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:12:14.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:12:14.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:14.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:14.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary7 2024-09-18T00:12:14.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd6 2024-09-18T00:12:14.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:12:14.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:12:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd7 2024-09-18T00:12:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:12:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:12:14.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:12:14.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 2846127d-142c-45b9-b096-a07f221235bd 2024-09-18T00:12:14.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:14.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:14.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:14.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:14 2024-09-18T00:12:14.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:14.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:12:14.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:14.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:14.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:14' 2024-09-18T00:12:14.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:14.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:14.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:14.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:14.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:14.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:14.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:14.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:14.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:14.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:14.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:14.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:14.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:14' 2024-09-18T00:12:14.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary8 up+unknown '2024-09-18 00:12:14' 2024-09-18T00:12:14.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:14.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:14.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:12:14.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:14.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:14' 2024-09-18T00:12:14.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:14.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:14.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:14.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:12:14.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 5cbf0aba-1bfb-4812-b772-7406d18bca14 2024-09-18T00:12:14.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:14.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:14.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:12:14.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:14 2024-09-18T00:12:14.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:12:14.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:12:14.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:12:14.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:12:14.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:12:14' 2024-09-18T00:12:14.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:12:14.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:12:14.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:12:14.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:12:14.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:12:14.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:14.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:12:14.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:14.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:12:14.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:12:14.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:12:14.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:12:14.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:14' 2024-09-18T00:12:14.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary7 up+unknown '2024-09-18 00:12:14' 2024-09-18T00:12:14.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:14.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:14.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:12:14.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:12:14.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:12:14' 2024-09-18T00:12:14.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:12:14.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:12:14.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:12:14.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING2842614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}2024-09-18 00:12:09image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"}2024-09-18 00:12:14image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}2024-09-18 00:12:14image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}2024-09-18 00:12:09image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:14image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:14image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}2024-09-18 00:12:14' 2024-09-18T00:12:14.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:14.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:12:14.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:14' 2024-09-18T00:12:14.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:12:14.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:14.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:14.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:14.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:14.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:14.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:14.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary8 2024-09-18T00:12:14.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:14.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:14.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:12:14.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:12:14.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary8 2024-09-18T00:12:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING2842614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}2024-09-18 00:12:09image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"}2024-09-18 00:12:14image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}2024-09-18 00:12:14image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}2024-09-18 00:12:09image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}2024-09-18 00:12:09image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:14image-primary82846127d-142c-45b9-b096-a07f221235bdup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:12:14image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:12:14cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}2024-09-18 00:12:14' 2024-09-18T00:12:14.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:12:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:12:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:12:14' 2024-09-18T00:12:14.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:12:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:12:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:12:14.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:12:14.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:12:14.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:14.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:12:14.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary7 2024-09-18T00:12:14.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:12:14.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:12:14.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:12:14.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:12:14.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary7 2024-09-18T00:12:15.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:12:15.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:12:15.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:12:15.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:12:15.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:12:15.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:12:16.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd8 2024-09-18T00:12:16.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:12:16.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:12:16.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd9 2024-09-18T00:12:16.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:12:16.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:13:34.145 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:13:34.145+0000 7fd4c158b640 -1 reset not still connected to 0x5643587075f0 2024-09-18T00:13:34.149 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:13:34.149+0000 7fd4c158b640 -1 reset not still connected to 0x564358707d40 2024-09-18T00:13:34.149 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:13:34.149+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a340 2024-09-18T00:13:34.149 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:13:34.149+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T00:13:34.149 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:13:34.149+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T00:13:34.195 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:13:34.198+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735aedd0 2024-09-18T00:13:51.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=d3e29ccb569c89f02743be8bf4d41e29 2024-09-18T00:13:51.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd0 2024-09-18T00:13:52.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=794c634cd3fe74e62ec69cadae508d46 2024-09-18T00:13:52.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd2 2024-09-18T00:13:53.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ d3e29ccb569c89f02743be8bf4d41e29 != \d\3\e\2\9\c\c\b\5\6\9\c\8\9\f\0\2\7\4\3\b\e\8\b\f\4\d\4\1\e\2\9 ]] 2024-09-18T00:13:54.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 794c634cd3fe74e62ec69cadae508d46 != \7\9\4\c\6\3\4\c\d\3\f\e\7\4\e\6\2\e\c\6\9\c\a\d\a\e\5\0\8\d\4\6 ]] 2024-09-18T00:13:54.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=c548cdf6a3f9777a69febac611824e2f 2024-09-18T00:13:54.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd4 2024-09-18T00:13:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ c548cdf6a3f9777a69febac611824e2f != \c\5\4\8\c\d\f\6\a\3\f\9\7\7\7\a\6\9\f\e\b\a\c\6\1\1\8\2\4\e\2\f ]] 2024-09-18T00:13:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:13:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 60008 2024-09-18T00:13:55.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:13:55.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 60009 2024-09-18T00:13:55.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=94c52a39d7d75b807f0b1d7bb64168f8 2024-09-18T00:13:55.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd1 2024-09-18T00:13:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=6452d440b68c7dde7b2ec5b8efcb9544 2024-09-18T00:13:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd3 2024-09-18T00:13:56.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 94c52a39d7d75b807f0b1d7bb64168f8 != \9\4\c\5\2\a\3\9\d\7\d\7\5\b\8\0\7\f\0\b\1\d\7\b\b\6\4\1\6\8\f\8 ]] 2024-09-18T00:13:56.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:13:56.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 60011 2024-09-18T00:13:56.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 6452d440b68c7dde7b2ec5b8efcb9544 != \6\4\5\2\d\4\4\0\b\6\8\c\7\d\d\e\7\b\2\e\c\5\b\8\e\f\c\b\9\5\4\4 ]] 2024-09-18T00:13:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=1d761a4f264508c80ad0f497b0c1c53b 2024-09-18T00:13:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd5 2024-09-18T00:13:59.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 1d761a4f264508c80ad0f497b0c1c53b != \1\d\7\6\1\a\4\f\2\6\4\5\0\8\c\8\0\a\d\0\f\4\9\7\b\0\c\1\c\5\3\b ]] 2024-09-18T00:14:00.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=9c52270816e3fc83308a4a7a9f0b2315 2024-09-18T00:14:00.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd6 2024-09-18T00:14:00.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=24709b11fb2630a9bbf4a7e0fa08f708 2024-09-18T00:14:00.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd7 2024-09-18T00:14:00.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=890c56486a23d7d06aa79071d2970819 2024-09-18T00:14:00.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd9 2024-09-18T00:14:01.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 9c52270816e3fc83308a4a7a9f0b2315 != \9\c\5\2\2\7\0\8\1\6\e\3\f\c\8\3\3\0\8\a\4\a\7\a\9\f\0\b\2\3\1\5 ]] 2024-09-18T00:14:01.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:14:01.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 60013 2024-09-18T00:14:01.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:14:01.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 60015 2024-09-18T00:14:01.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:14:01.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 60017 2024-09-18T00:14:01.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=cac5b70057e374f548dc6ead679a7b2a 2024-09-18T00:14:01.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd8 2024-09-18T00:14:01.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 24709b11fb2630a9bbf4a7e0fa08f708 != \2\4\7\0\9\b\1\1\f\b\2\6\3\0\a\9\b\b\f\4\a\7\e\0\f\a\0\8\f\7\0\8 ]] 2024-09-18T00:14:01.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 890c56486a23d7d06aa79071d2970819 != \8\9\0\c\5\6\4\8\6\a\2\3\d\7\d\0\6\a\a\7\9\0\7\1\d\2\9\7\0\8\1\9 ]] 2024-09-18T00:14:01.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ cac5b70057e374f548dc6ead679a7b2a != \c\a\c\5\b\7\0\0\5\7\e\3\7\4\f\5\4\8\d\c\6\e\a\d\6\7\9\a\7\b\2\a ]] 2024-09-18T00:14:01.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:14:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 60019 2024-09-18T00:14:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:14:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 60021 2024-09-18T00:14:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:14:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 60022 2024-09-18T00:14:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:14:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:14:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary1 2024-09-18T00:14:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:14:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+replaying 2024-09-18T00:14:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:01.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:14:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+replaying '' '' 2024-09-18T00:14:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:02.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:02.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:14:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:14:02.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: e748bb2a-a5aa-46c1-b88f-a4adee5d24c6 2024-09-18T00:14:02.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:14:02.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60948480.0,"last_snapshot_bytes":60948480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618327,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"} 2024-09-18T00:14:02.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:14:02.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:13:39 2024-09-18T00:14:02.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:14:02.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:14:02.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:14:02.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:14:02.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:13:59' 2024-09-18T00:14:02.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:14:02.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:14:02.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:14:02.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60948480.0,"last_snapshot_bytes":60948480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618327,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"} 2024-09-18T00:14:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:14:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:02.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:14:02.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:02.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:14:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:14:02.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:14:02.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:14:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:13:39' 2024-09-18T00:14:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+replaying '2024-09-18 00:13:39' 2024-09-18T00:14:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:13:39' 2024-09-18T00:14:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:14:03.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1e748bb2a-a5aa-46c1-b88f-a4adee5d24c6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60948480.0,"last_snapshot_bytes":60948480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618327,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:13:59image-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60366848.0,"last_snapshot_bytes":60366848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618331,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:13:59image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72425472.0,"last_snapshot_bytes":72425472,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:13:59image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72380416.0,"last_snapshot_bytes":72380416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:13:59image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:13:59image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72355840.0,"last_snapshot_bytes":72355840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618327,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:13:59image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70139904.0,"last_snapshot_bytes":70139904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:13:59image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63979520.0,"last_snapshot_bytes":63979520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:13:59image-primary82846127d-142c-45b9-b096-a07f221235bdup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70017024.0,"last_snapshot_bytes":70017024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:13:59image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70012928.0,"last_snapshot_bytes":70012928,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:13:59' 2024-09-18T00:14:03.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:14:03.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:14:03.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:13:39' 2024-09-18T00:14:03.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:14:03.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:14:03.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:03.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:14:03.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:03.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:03.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:03.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary1 2024-09-18T00:14:03.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:14:03.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:03.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:03.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary1 2024-09-18T00:14:03.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary1 2024-09-18T00:14:05.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:14:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary1 2024-09-18T00:14:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:05.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:05.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:14:05.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary1 2024-09-18T00:14:05.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:05.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary1 2024-09-18T00:14:05.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:05.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:14:05.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary2 2024-09-18T00:14:05.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:05.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:05.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:05.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:14:05.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+replaying 2024-09-18T00:14:05.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:05.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:05.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:05.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:05.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:05.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:05.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:05.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:14:06.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+replaying '' '' 2024-09-18T00:14:06.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:06.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:06.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:06.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:06.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:06.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:06.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:06.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:06.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:14:06.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:14:06.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f00bdbdc-f2bd-4210-80a6-7c7769eada40 2024-09-18T00:14:06.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:14:06.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72425472.0,"last_snapshot_bytes":72425472,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"} 2024-09-18T00:14:06.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:14:06.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:13:39 2024-09-18T00:14:06.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:14:06.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:14:06.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:14:06.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:14:06.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:04' 2024-09-18T00:14:06.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:14:06.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:14:06.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:14:06.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72425472.0,"last_snapshot_bytes":72425472,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"} 2024-09-18T00:14:06.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:14:06.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:06.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:14:06.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:06.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:14:06.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:14:06.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:14:06.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:14:06.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:13:39' 2024-09-18T00:14:06.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+replaying '2024-09-18 00:13:39' 2024-09-18T00:14:06.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:06.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:06.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:06.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:06.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:13:39' 2024-09-18T00:14:06.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:06.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:06.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:14:06.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60366848.0,"last_snapshot_bytes":60366848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618331,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:04image-primary2f00bdbdc-f2bd-4210-80a6-7c7769eada40up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72425472.0,"last_snapshot_bytes":72425472,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:04image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72380416.0,"last_snapshot_bytes":72380416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:04image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:04image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72355840.0,"last_snapshot_bytes":72355840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618327,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:04image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70139904.0,"last_snapshot_bytes":70139904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:04image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63979520.0,"last_snapshot_bytes":63979520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:04image-primary82846127d-142c-45b9-b096-a07f221235bdup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70017024.0,"last_snapshot_bytes":70017024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:04image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70012928.0,"last_snapshot_bytes":70012928,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:13:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:04' 2024-09-18T00:14:06.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:14:06.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:14:06.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:13:39' 2024-09-18T00:14:06.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:14:06.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:14:06.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:06.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:14:06.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:06.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:06.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:06.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary2 2024-09-18T00:14:06.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:14:06.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:06.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:06.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary2 2024-09-18T00:14:06.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary2 2024-09-18T00:14:08.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:14:08.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary2 2024-09-18T00:14:08.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:08.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:08.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:08.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:08.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:14:08.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary2 2024-09-18T00:14:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary2 2024-09-18T00:14:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary3 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+replaying 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:08.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:08.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:08.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:14:09.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+replaying '' '' 2024-09-18T00:14:09.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:09.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:09.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:09.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:09.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:09.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:09.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:09.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:09.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 49280c03-8d1c-4c44-b8cc-dc519e871b86 2024-09-18T00:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72380416.0,"last_snapshot_bytes":72380416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"} 2024-09-18T00:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:09 2024-09-18T00:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:09' 2024-09-18T00:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:14:09.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:14:09.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:14:09.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72380416.0,"last_snapshot_bytes":72380416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"} 2024-09-18T00:14:09.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:14:09.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:09.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:14:09.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:09.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:14:09.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:14:09.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:14:09.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:14:09.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:09.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+replaying '2024-09-18 00:14:09' 2024-09-18T00:14:09.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:09.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:09.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:09.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:09.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:14:09' 2024-09-18T00:14:09.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:09.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:09.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:14:09.346 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:09.350+0000 7f849c00d640 -1 reset not still connected to 0x5629b8b7ad00 2024-09-18T00:14:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK842544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60366848.0,"last_snapshot_bytes":60366848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618331,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary349280c03-8d1c-4c44-b8cc-dc519e871b86up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72380416.0,"last_snapshot_bytes":72380416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72355840.0,"last_snapshot_bytes":72355840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618327,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70139904.0,"last_snapshot_bytes":70139904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63979520.0,"last_snapshot_bytes":63979520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary82846127d-142c-45b9-b096-a07f221235bdup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70017024.0,"last_snapshot_bytes":70017024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70012928.0,"last_snapshot_bytes":70012928,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09' 2024-09-18T00:14:09.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:14:09.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:14:09.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:14:09.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:14:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:14:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary3 2024-09-18T00:14:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:14:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary3 2024-09-18T00:14:09.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary3 2024-09-18T00:14:11.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:14:11.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary3 2024-09-18T00:14:11.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:11.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:11.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:11.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:11.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:14:11.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary3 2024-09-18T00:14:11.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:11.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary3 2024-09-18T00:14:11.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:11.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary4 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+replaying 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:14:12.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+replaying '' '' 2024-09-18T00:14:12.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:12.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:12.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:12.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:12.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:12.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:12.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:12.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:14:12.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:14:12.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3 2024-09-18T00:14:12.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"} 2024-09-18T00:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:09 2024-09-18T00:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:09' 2024-09-18T00:14:12.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:14:12.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:14:12.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:14:12.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"} 2024-09-18T00:14:12.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:14:12.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:12.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:14:12.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:12.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:14:12.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:14:12.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:14:12.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:14:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+replaying '2024-09-18 00:14:09' 2024-09-18T00:14:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:14:09' 2024-09-18T00:14:12.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:12.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:12.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:14:12.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60366848.0,"last_snapshot_bytes":60366848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618331,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary4b5b70b2a-a2f7-49af-b25d-3ff686d3b1f3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72351744.0,"last_snapshot_bytes":72351744,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72355840.0,"last_snapshot_bytes":72355840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618327,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70139904.0,"last_snapshot_bytes":70139904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63979520.0,"last_snapshot_bytes":63979520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary82846127d-142c-45b9-b096-a07f221235bdup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70017024.0,"last_snapshot_bytes":70017024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70012928.0,"last_snapshot_bytes":70012928,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:09' 2024-09-18T00:14:12.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:14:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:14:12.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:12.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:14:12.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:14:12.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:12.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:14:12.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:12.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:12.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:12.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary4 2024-09-18T00:14:12.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:14:12.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:12.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:12.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary4 2024-09-18T00:14:12.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary4 2024-09-18T00:14:14.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:14:14.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary4 2024-09-18T00:14:14.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:14.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:14.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:14.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:14.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:14:14.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary4 2024-09-18T00:14:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:14.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary4 2024-09-18T00:14:14.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary5 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+replaying 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:14.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:14.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:14.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:14:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+replaying '' '' 2024-09-18T00:14:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:15.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:15.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:15.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:15.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:14:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:14:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: fac94502-f15a-4e8a-b812-20cd1df947cf 2024-09-18T00:14:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:14:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72355840.0,"last_snapshot_bytes":72355840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618327,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"} 2024-09-18T00:14:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:14:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:09 2024-09-18T00:14:15.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:14:15.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:14:15.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:14:15.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:14:15.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:14' 2024-09-18T00:14:15.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:14:15.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:14:15.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:14:15.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72355840.0,"last_snapshot_bytes":72355840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618327,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"} 2024-09-18T00:14:15.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:14:15.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:15.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:14:15.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:15.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:14:15.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:14:15.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:14:15.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:14:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+replaying '2024-09-18 00:14:09' 2024-09-18T00:14:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:15.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:14:09' 2024-09-18T00:14:15.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:15.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:15.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK642544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60366848.0,"last_snapshot_bytes":60366848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618331,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:14image-primary5fac94502-f15a-4e8a-b812-20cd1df947cfup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72355840.0,"last_snapshot_bytes":72355840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618327,"remote_snapshot_timestamp":1726618327,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:14image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70139904.0,"last_snapshot_bytes":70139904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:14image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63979520.0,"last_snapshot_bytes":63979520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:14image-primary82846127d-142c-45b9-b096-a07f221235bdup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70017024.0,"last_snapshot_bytes":70017024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:14image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70012928.0,"last_snapshot_bytes":70012928,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:14' 2024-09-18T00:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:14:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:14:15.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:15.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:14:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:14:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:14:15.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary5 2024-09-18T00:14:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:14:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:15.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary5 2024-09-18T00:14:15.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary5 2024-09-18T00:14:17.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:14:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary5 2024-09-18T00:14:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:14:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary5 2024-09-18T00:14:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:17.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary5 2024-09-18T00:14:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:14:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary6 2024-09-18T00:14:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:17.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:14:17.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+replaying 2024-09-18T00:14:17.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:17.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:17.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:17.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:17.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:17.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:17.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:17.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:14:18.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+replaying '' '' 2024-09-18T00:14:18.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:18.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:14:18.502 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:18.505+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:14:18.502 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:18.505+0000 7f849c00d640 -1 reset not still connected to 0x5629b4e6dad0 2024-09-18T00:14:18.502 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:18.505+0000 7f849c00d640 -1 reset not still connected to 0x5629b8bedad0 2024-09-18T00:14:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:14:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: e7927f4e-32c4-4eaa-be90-8922d3a6eb2f 2024-09-18T00:14:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:14:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70139904.0,"last_snapshot_bytes":70139904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"} 2024-09-18T00:14:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:14:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:09 2024-09-18T00:14:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:14:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:14:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:14:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:14:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:14' 2024-09-18T00:14:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:14:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:14:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:14:18.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70139904.0,"last_snapshot_bytes":70139904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"} 2024-09-18T00:14:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:14:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:14:18.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:18.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:14:18.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:14:18.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:14:18.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:14:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+replaying '2024-09-18 00:14:09' 2024-09-18T00:14:18.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:18.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:18.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:18.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:18.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:14:09' 2024-09-18T00:14:18.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:18.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:18.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:14:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60366848.0,"last_snapshot_bytes":60366848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618331,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:14image-primary6e7927f4e-32c4-4eaa-be90-8922d3a6eb2fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70139904.0,"last_snapshot_bytes":70139904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618326,"remote_snapshot_timestamp":1726618326,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:14image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63979520.0,"last_snapshot_bytes":63979520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:14image-primary82846127d-142c-45b9-b096-a07f221235bdup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70017024.0,"last_snapshot_bytes":70017024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:14image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70012928.0,"last_snapshot_bytes":70012928,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:14' 2024-09-18T00:14:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:14:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:14:18.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:18.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:14:18.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:14:18.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:18.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:14:18.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary6 2024-09-18T00:14:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:14:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary6 2024-09-18T00:14:18.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary6 2024-09-18T00:14:20.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:14:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary6 2024-09-18T00:14:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:14:20.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary6 2024-09-18T00:14:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary6 2024-09-18T00:14:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:14:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary7 2024-09-18T00:14:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:14:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+replaying 2024-09-18T00:14:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:14:21.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+replaying '' '' 2024-09-18T00:14:21.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:14:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:14:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 5cbf0aba-1bfb-4812-b772-7406d18bca14 2024-09-18T00:14:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:14:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63979520.0,"last_snapshot_bytes":63979520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"} 2024-09-18T00:14:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:14:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:09 2024-09-18T00:14:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:14:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:14:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:14:21.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:14:21.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:19' 2024-09-18T00:14:21.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:14:21.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:14:21.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:14:21.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63979520.0,"last_snapshot_bytes":63979520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"} 2024-09-18T00:14:21.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:14:21.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:21.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:14:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:14:21.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:14:21.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:14:21.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:14:21.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:21.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+replaying '2024-09-18 00:14:09' 2024-09-18T00:14:21.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:21.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:21.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:21.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:21.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:14:09' 2024-09-18T00:14:21.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:21.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:21.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:14:21.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60366848.0,"last_snapshot_bytes":60366848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618331,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:19image-primary75cbf0aba-1bfb-4812-b772-7406d18bca14up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63979520.0,"last_snapshot_bytes":63979520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:19image-primary82846127d-142c-45b9-b096-a07f221235bdup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70017024.0,"last_snapshot_bytes":70017024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:19image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70012928.0,"last_snapshot_bytes":70012928,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:19' 2024-09-18T00:14:21.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:14:21.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:14:21.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:21.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:14:21.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:14:21.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:21.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:14:21.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:21.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:21.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:21.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary7 2024-09-18T00:14:21.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:14:21.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:21.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:21.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary7 2024-09-18T00:14:21.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary7 2024-09-18T00:14:23.165 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:23.165+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:14:23.165 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:23.165+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:14:23.165 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:23.165+0000 7f849c00d640 -1 reset not still connected to 0x5629b4e6dad0 2024-09-18T00:14:23.168 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:23.165+0000 7f849c00d640 -1 reset not still connected to 0x5629b8bedad0 2024-09-18T00:14:23.168 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:23.169+0000 7f849c00d640 -1 reset not still connected to 0x5629dcefd6c0 2024-09-18T00:14:23.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:14:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary7 2024-09-18T00:14:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:14:23.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary7 2024-09-18T00:14:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary7 2024-09-18T00:14:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:14:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary8 2024-09-18T00:14:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:14:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+replaying 2024-09-18T00:14:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:14:24.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+replaying '' '' 2024-09-18T00:14:24.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:24.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:24.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:24.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:24.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:24.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:24.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:24.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:24.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:14:24.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 2846127d-142c-45b9-b096-a07f221235bd 2024-09-18T00:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70017024.0,"last_snapshot_bytes":70017024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"} 2024-09-18T00:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:09 2024-09-18T00:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:24' 2024-09-18T00:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:14:24.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:14:24.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:14:24.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70017024.0,"last_snapshot_bytes":70017024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"} 2024-09-18T00:14:24.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:14:24.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:24.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:14:24.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:24.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:14:24.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:14:24.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:14:24.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:14:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+replaying '2024-09-18 00:14:09' 2024-09-18T00:14:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:14:09' 2024-09-18T00:14:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:14:24.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK342544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60366848.0,"last_snapshot_bytes":60366848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618331,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:24image-primary82846127d-142c-45b9-b096-a07f221235bdup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70017024.0,"last_snapshot_bytes":70017024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618335,"remote_snapshot_timestamp":1726618335,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:24image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70012928.0,"last_snapshot_bytes":70012928,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:24' 2024-09-18T00:14:24.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:14:24.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:14:24.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:24.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:14:24.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:14:24.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:24.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:14:24.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:24.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:24.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:24.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary8 2024-09-18T00:14:24.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:14:24.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:24.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:24.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary8 2024-09-18T00:14:24.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary8 2024-09-18T00:14:26.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:14:26.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary8 2024-09-18T00:14:26.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:26.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:26.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:26.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:26.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:14:26.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary8 2024-09-18T00:14:26.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:26.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary8 2024-09-18T00:14:26.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:26.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:14:26.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary9 2024-09-18T00:14:26.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:14:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+replaying 2024-09-18T00:14:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:26.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:14:26.459 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:26.461+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:14:26.460 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:26.465+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:14:26.460 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:26.465+0000 7f849c00d640 -1 reset not still connected to 0x5629aad18f70 2024-09-18T00:14:26.460 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:26.465+0000 7f849c00d640 -1 reset not still connected to 0x5629b4e6dad0 2024-09-18T00:14:26.460 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:14:26.465+0000 7f849c00d640 -1 reset not still connected to 0x5629b8bedad0 2024-09-18T00:14:27.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+replaying '' '' 2024-09-18T00:14:27.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:27.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:27.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:27.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:27.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:27.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:27.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:27.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:27.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:14:27.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:14:27.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: bcd32a3a-52fc-4137-aa0a-509317d6523f 2024-09-18T00:14:27.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:14:27.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70012928.0,"last_snapshot_bytes":70012928,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"} 2024-09-18T00:14:27.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:14:27.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:09 2024-09-18T00:14:27.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:14:27.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:14:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:14:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:14:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:24' 2024-09-18T00:14:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:14:27.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:14:27.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:14:27.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70012928.0,"last_snapshot_bytes":70012928,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"} 2024-09-18T00:14:27.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:14:27.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:27.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:14:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:14:27.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:14:27.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:14:27.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:14:27.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:27.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+replaying '2024-09-18 00:14:09' 2024-09-18T00:14:27.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:27.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:27.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:27.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:27.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:14:09' 2024-09-18T00:14:27.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:27.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:27.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:14:27.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60366848.0,"last_snapshot_bytes":60366848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618331,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:24image-primary9bcd32a3a-52fc-4137-aa0a-509317d6523fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70012928.0,"last_snapshot_bytes":70012928,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618333,"remote_snapshot_timestamp":1726618333,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:24' 2024-09-18T00:14:27.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:14:27.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:14:27.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:27.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:14:27.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:14:27.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:27.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:14:27.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:27.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:27.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:27.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary9 2024-09-18T00:14:27.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:14:27.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:27.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:27.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary9 2024-09-18T00:14:27.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary9 2024-09-18T00:14:29.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:14:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary9 2024-09-18T00:14:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:14:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary9 2024-09-18T00:14:29.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:29.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary9 2024-09-18T00:14:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:14:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary10 2024-09-18T00:14:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:14:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+replaying 2024-09-18T00:14:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:29.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:14:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+replaying '' '' 2024-09-18T00:14:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:14:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:14:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:14:30.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:14:30.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 02b36501-4523-474d-b4d9-f59723d6b12e 2024-09-18T00:14:30.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:14:30.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60366848.0,"last_snapshot_bytes":60366848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618331,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"} 2024-09-18T00:14:30.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:14:30.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:09 2024-09-18T00:14:30.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:14:30.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:14:30.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:14:30.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:14:30.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:14:29' 2024-09-18T00:14:30.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:14:30.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:14:30.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:14:30.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60366848.0,"last_snapshot_bytes":60366848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618331,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"} 2024-09-18T00:14:30.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:14:30.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:30.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:14:30.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:30.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:14:30.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:14:30.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:14:30.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:14:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+replaying '2024-09-18 00:14:09' 2024-09-18T00:14:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:30.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:14:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:14:09' 2024-09-18T00:14:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:14:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:14:30.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:14:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1002b36501-4523-474d-b4d9-f59723d6b12eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60366848.0,"last_snapshot_bytes":60366848,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618331,"remote_snapshot_timestamp":1726618331,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:14:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:14:29' 2024-09-18T00:14:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:14:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:14:30.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:14:09' 2024-09-18T00:14:30.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:14:30.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:14:30.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:14:30.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:14:30.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:14:30.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:30.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:30.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary10 2024-09-18T00:14:30.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:14:30.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:30.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:30.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary10 2024-09-18T00:14:30.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary10 2024-09-18T00:14:32.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:14:32.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary10 2024-09-18T00:14:32.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:32.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:32.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:32.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:14:32.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:14:32.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary10 2024-09-18T00:14:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:32.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary10 2024-09-18T00:14:32.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..10} 2024-09-18T00:14:32.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS=() 2024-09-18T00:14:32.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS=() 2024-09-18T00:14:32.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS=() 2024-09-18T00:14:32.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS=() 2024-09-18T00:14:32.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RET=0 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary1 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary1 snapshot 10G 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:32.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary1 10G 2024-09-18T00:14:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:14:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:14:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:14:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:32.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary1 2024-09-18T00:14:32.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:14:32.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:32.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:14:32.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:32.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary1 snapshot 2024-09-18T00:14:32.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:32.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:32.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:32.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:32.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary1 snapshot 2024-09-18T00:14:33.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:14:33.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary1 2024-09-18T00:14:33.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary1': 2024-09-18T00:14:33.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:14:33.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:14:33.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:14:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1e3754fe99ba 2024-09-18T00:14:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1e3754fe99ba 2024-09-18T00:14:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:14:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:14:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:14:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:14:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:14:32 2024 2024-09-18T00:14:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:14:32 2024 2024-09-18T00:14:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:14:32 2024 2024-09-18T00:14:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:14:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:14:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 684b85b9-e82c-49e4-af6b-eb72455a5948 2024-09-18T00:14:33.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:14:33.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:14:33.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:14:33.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd0 2024-09-18T00:14:33.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:14:33.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd0 2024-09-18T00:14:33.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:14:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:14:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:14:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: f0aaee13-bc4f-4b9b-a8d1-3292ad239db2 2024-09-18T00:14:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:14:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:14:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:14:33.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:14:33.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:14:33.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:14:33.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:33.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary1 2024-09-18T00:14:33.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd0 test-primary1 2024-09-18T00:14:34.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:14:34.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary1 2024-09-18T00:14:34.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:14:34.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary1 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary1 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary2 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary2 snapshot 10G 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary1 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary1 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary2 10G 2024-09-18T00:14:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary2 2024-09-18T00:14:34.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:14:34.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:34.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:14:34.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:34.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary2 snapshot 2024-09-18T00:14:34.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:34.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:34.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:34.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:34.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary2 snapshot 2024-09-18T00:14:34.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 994 2024-09-18T00:14:34.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:14:34.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:34.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary2 2024-09-18T00:14:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary2': 2024-09-18T00:14:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:14:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:14:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:14:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1e472ac0779 2024-09-18T00:14:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1e472ac0779 2024-09-18T00:14:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:14:34.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:14:34.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:14:34.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:14:34.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:14:34 2024 2024-09-18T00:14:34.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:14:34 2024 2024-09-18T00:14:34.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:14:34 2024 2024-09-18T00:14:34.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:14:34.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:14:34.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4 2024-09-18T00:14:34.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:14:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:14:34.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:14:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd1 2024-09-18T00:14:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:14:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd1 2024-09-18T00:14:34.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:14:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:14:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:14:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 88d7452d-bfe3-47c4-8f41-67f3c5c1daa2 2024-09-18T00:14:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:14:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:14:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:14:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:14:34.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:14:34.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:14:34.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:34.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary2 2024-09-18T00:14:34.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd1 test-primary2 2024-09-18T00:14:34.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:14:34.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary1/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary1' 2024-09-18T00:14:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:14:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary2 2024-09-18T00:14:35.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:14:35.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary2 2024-09-18T00:14:35.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary2 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary3 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary3 snapshot 10G 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:35.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary2 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary2 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary3 10G 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:35.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:35.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:14:35.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:14:35.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:14:35.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:14:35.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:35.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary3 2024-09-18T00:14:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:14:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:14:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary3 snapshot 2024-09-18T00:14:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary3 snapshot 2024-09-18T00:14:35.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:14:35.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary2/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary2' 2024-09-18T00:14:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 996 2024-09-18T00:14:35.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:14:35.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:35.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary3 2024-09-18T00:14:35.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary3': 2024-09-18T00:14:35.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1e569269deb7 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1e569269deb7 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:14:35 2024 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:14:35 2024 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:14:35 2024 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:14:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:14:35.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 74d7ffd9-fbb0-4520-9d48-784c6cd85819 2024-09-18T00:14:35.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:14:35.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:14:35.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:14:35.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd2 2024-09-18T00:14:35.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:14:35.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd2 2024-09-18T00:14:35.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:14:35.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:14:35.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:14:35.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 39a8463d-8310-4f70-b851-fcd4c0f4f0e8 2024-09-18T00:14:35.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:14:35.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:14:35.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:35.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:14:35.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:14:35.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:14:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:14:36.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary3 2024-09-18T00:14:36.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd2 test-primary3 2024-09-18T00:14:36.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:14:36.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary3 2024-09-18T00:14:36.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:14:36.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary3 2024-09-18T00:14:36.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:14:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:14:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:14:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary3 2024-09-18T00:14:36.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary4 2024-09-18T00:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary4 snapshot 10G 2024-09-18T00:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary3 2024-09-18T00:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary3 2024-09-18T00:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:14:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary4 10G 2024-09-18T00:14:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:14:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:14:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:14:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:36.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary4 2024-09-18T00:14:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary4 snapshot 2024-09-18T00:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary4 snapshot 2024-09-18T00:14:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:14:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary3/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary3' 2024-09-18T00:14:37.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:14:37.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 998 2024-09-18T00:14:37.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:37.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary4 2024-09-18T00:14:37.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:37.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:14:37.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:37.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:37.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:37.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:14:37.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary4': 2024-09-18T00:14:37.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:14:37.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:14:37.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:14:37.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1e66811d283 2024-09-18T00:14:37.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1e66811d283 2024-09-18T00:14:37.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:14:37.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:14:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:14:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:14:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:14:36 2024 2024-09-18T00:14:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:14:36 2024 2024-09-18T00:14:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:14:36 2024 2024-09-18T00:14:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:14:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:14:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 158b22b4-54b8-411f-9e7e-2163182393ca 2024-09-18T00:14:37.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:14:37.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:14:37.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:14:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd3 2024-09-18T00:14:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:14:37.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd3 2024-09-18T00:14:37.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:14:37.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:14:37.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:14:37.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 994b59fe-b353-4cd4-b3f6-dce4fff3caa8 2024-09-18T00:14:37.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:14:37.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:14:37.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:37.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:14:37.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:14:38.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:14:38.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:14:38.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:38.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary4 2024-09-18T00:14:38.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd3 test-primary4 2024-09-18T00:14:38.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:14:38.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary4 2024-09-18T00:14:38.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:14:38.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary4 2024-09-18T00:14:38.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:38.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:14:38.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:38.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:38.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:38.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary4 2024-09-18T00:14:38.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:38.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:14:38.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary5 2024-09-18T00:14:38.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:14:38.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary5 snapshot 10G 2024-09-18T00:14:38.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:38.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:38.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:38.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:38.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:38.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:38.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:38.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:38.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary4 2024-09-18T00:14:38.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:38.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:38.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:14:38.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:14:38.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary4 2024-09-18T00:14:38.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:38.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary5 10G 2024-09-18T00:14:38.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:38.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:38.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:38.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:38.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:38.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:38.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:14:38.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:14:38.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:14:38.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:38.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary5 2024-09-18T00:14:38.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:14:38.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:38.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:14:38.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:38.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary5 snapshot 2024-09-18T00:14:38.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:38.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:38.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:38.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:38.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary5 snapshot 2024-09-18T00:14:38.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:38.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:14:38.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:38.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:38.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:38.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:14:38.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:14:38.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary4/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary4' 2024-09-18T00:14:39.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1000 2024-09-18T00:14:39.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:39.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:14:39.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1001 2024-09-18T00:14:39.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary5 2024-09-18T00:14:39.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:39.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary5': 2024-09-18T00:14:39.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:14:39.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1e78cbe56ad6 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1e78cbe56ad6 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:14:38 2024 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:14:38 2024 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:14:38 2024 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:14:39.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 8d545589-7b5b-44e3-b6e8-4df0695358a6 2024-09-18T00:14:39.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:14:39.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:14:39.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:14:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd4 2024-09-18T00:14:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:14:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd4 2024-09-18T00:14:39.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:14:39.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:14:39.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:14:39.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 99177e86-e860-412b-ad70-f12a43d620a8 2024-09-18T00:14:39.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:14:39.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:14:39.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:39.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:14:39.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:14:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:14:40.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:14:40.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:40.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary5 2024-09-18T00:14:40.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd4 test-primary5 2024-09-18T00:14:40.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:14:40.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary5 2024-09-18T00:14:40.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary5 2024-09-18T00:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary5 2024-09-18T00:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary6 2024-09-18T00:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:14:40.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary6 snapshot 10G 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary5 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary5 2024-09-18T00:14:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary6 10G 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary6 2024-09-18T00:14:40.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:40.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:14:40.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:40.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:40.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:40.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:14:40.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1003 2024-09-18T00:14:40.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:14:40.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:40.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:14:40.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:40.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary6 snapshot 2024-09-18T00:14:40.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:40.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:40.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:40.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:40.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary6 snapshot 2024-09-18T00:14:40.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:40.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:14:40.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary5/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary5' 2024-09-18T00:14:41.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:14:41.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1004 2024-09-18T00:14:41.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:41.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary6 2024-09-18T00:14:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary6': 2024-09-18T00:14:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:14:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:14:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:14:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1e8845d8b4b9 2024-09-18T00:14:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1e8845d8b4b9 2024-09-18T00:14:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:14:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:14:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:14:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:14:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:14:40 2024 2024-09-18T00:14:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:14:40 2024 2024-09-18T00:14:41.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:14:40 2024 2024-09-18T00:14:41.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:14:41.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:14:41.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 88c5b661-6663-4bbd-986c-9e5a7fb24308 2024-09-18T00:14:41.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:14:41.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:14:41.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:14:41.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd5 2024-09-18T00:14:41.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:14:41.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd5 2024-09-18T00:14:41.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:14:41.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:14:41.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:14:41.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 043e7baa-9d5c-4e43-a090-7e69625b5216 2024-09-18T00:14:41.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:14:41.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:14:41.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:41.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:14:41.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:14:42.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:14:42.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:14:42.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:42.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary6 2024-09-18T00:14:42.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd5 test-primary6 2024-09-18T00:14:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1005 2024-09-18T00:14:42.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:14:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary6 2024-09-18T00:14:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:14:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary6 2024-09-18T00:14:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:14:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary6 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary6 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary7 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary7 snapshot 10G 2024-09-18T00:14:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary6 2024-09-18T00:14:42.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:42.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:42.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:42.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:42.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:42.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:42.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:42.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:14:42.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:42.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary7 10G 2024-09-18T00:14:42.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:42.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:14:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:14:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:14:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary7 2024-09-18T00:14:42.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:14:42.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:42.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:14:42.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:42.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary7 snapshot 2024-09-18T00:14:42.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:42.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:42.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:42.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:42.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary7 snapshot 2024-09-18T00:14:42.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:42.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:14:42.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:42.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:42.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:42.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:14:42.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:14:42.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary6/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary6' 2024-09-18T00:14:42.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:42.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:14:42.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:42.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:42.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:42.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:14:43.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:43.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:14:43.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:43.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:43.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:43.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:14:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:14:43.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1007 2024-09-18T00:14:43.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:43.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary7 2024-09-18T00:14:43.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary7': 2024-09-18T00:14:43.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:14:43.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:14:43.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:14:43.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1e9eac23c4d7 2024-09-18T00:14:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1e9eac23c4d7 2024-09-18T00:14:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:14:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:14:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:14:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:14:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:14:42 2024 2024-09-18T00:14:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:14:42 2024 2024-09-18T00:14:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:14:42 2024 2024-09-18T00:14:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:14:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:14:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 79b8265e-910e-4b6e-9e5c-df55997a9d65 2024-09-18T00:14:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:14:43.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:14:43.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:14:43.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd6 2024-09-18T00:14:43.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:14:43.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd6 2024-09-18T00:14:43.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:14:43.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:14:43.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:14:43.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 252e69a5-fc29-476a-a703-cd67f60247d7 2024-09-18T00:14:43.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:14:43.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:14:43.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:43.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:14:43.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:14:44.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:14:44.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:14:44.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:44.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary7 2024-09-18T00:14:44.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd6 test-primary7 2024-09-18T00:14:44.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1010 2024-09-18T00:14:44.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1009 2024-09-18T00:14:44.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:44.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:44.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:14:44.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:44.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:44.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:44.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:14:44.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:14:44.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary7 2024-09-18T00:14:44.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:14:44.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary7 2024-09-18T00:14:44.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:14:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:14:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary8 2024-09-18T00:14:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary8 snapshot 10G 2024-09-18T00:14:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary7 2024-09-18T00:14:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:14:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary8 10G 2024-09-18T00:14:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:14:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary7 2024-09-18T00:14:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:14:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:14:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary7 2024-09-18T00:14:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary8 2024-09-18T00:14:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:14:44.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:44.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:44.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:44.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:14:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:14:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:14:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary8 snapshot 2024-09-18T00:14:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:44.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:44.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:44.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:44.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary8 snapshot 2024-09-18T00:14:44.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:14:44.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary7/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary7' 2024-09-18T00:14:45.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:45.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:14:45.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:45.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:45.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:45.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:14:45.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1011 2024-09-18T00:14:45.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:45.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:14:45.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary8 2024-09-18T00:14:45.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1013 2024-09-18T00:14:45.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:45.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary8': 2024-09-18T00:14:45.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:14:45.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:14:45.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:14:45.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1eb53b094a5 2024-09-18T00:14:45.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1eb53b094a5 2024-09-18T00:14:45.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:14:45.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:14:45.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:14:45.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:14:45.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:14:44 2024 2024-09-18T00:14:45.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:14:44 2024 2024-09-18T00:14:45.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:14:44 2024 2024-09-18T00:14:45.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:14:45.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:14:45.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 59ac8874-602c-4836-b596-7e605198b3a1 2024-09-18T00:14:45.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:14:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:14:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:14:45.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd7 2024-09-18T00:14:45.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:14:45.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd7 2024-09-18T00:14:45.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:14:45.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:14:45.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:14:45.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 3fcd2caf-d2bb-4e82-9121-c497f1e6b1fc 2024-09-18T00:14:45.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:14:45.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:14:45.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:45.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:14:45.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:14:46.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:14:46.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:14:46.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:46.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary8 2024-09-18T00:14:46.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd7 test-primary8 2024-09-18T00:14:46.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1012 2024-09-18T00:14:46.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:46.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:14:46.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary8 2024-09-18T00:14:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:14:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary8 2024-09-18T00:14:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:14:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:14:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary9 2024-09-18T00:14:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary8 2024-09-18T00:14:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary9 snapshot 10G 2024-09-18T00:14:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:14:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:14:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary9 10G 2024-09-18T00:14:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:14:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary8 2024-09-18T00:14:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary8 2024-09-18T00:14:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:14:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:14:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:14:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary9 2024-09-18T00:14:46.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:46.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:14:46.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:46.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:46.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:46.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:14:46.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:14:46.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:46.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:14:46.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:46.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary9 snapshot 2024-09-18T00:14:46.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:46.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:46.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:46.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:46.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary9 snapshot 2024-09-18T00:14:46.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:14:46.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary8/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary8' 2024-09-18T00:14:47.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1015 2024-09-18T00:14:47.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:47.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1016 2024-09-18T00:14:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:14:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:14:47.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:47.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:14:47.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:47.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:14:47.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:47.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:47.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:14:47.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary9 2024-09-18T00:14:47.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary9': 2024-09-18T00:14:47.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:14:47.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:14:47.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:14:47.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1eca270c549e 2024-09-18T00:14:47.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1eca270c549e 2024-09-18T00:14:47.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:14:47.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:14:46 2024 2024-09-18T00:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:14:46 2024 2024-09-18T00:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:14:46 2024 2024-09-18T00:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 0c2b99dd-43bc-4302-bae9-69b6a68f20e6 2024-09-18T00:14:47.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:14:47.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:14:47.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:14:47.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd8 2024-09-18T00:14:47.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:14:47.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd8 2024-09-18T00:14:47.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:14:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:14:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:14:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 9ae3f605-29c7-4188-b4a8-8340790aabe3 2024-09-18T00:14:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:14:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:14:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:14:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:14:47.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:14:48.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:14:48.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary9 2024-09-18T00:14:48.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd8 test-primary9 2024-09-18T00:14:48.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:14:48.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary9 2024-09-18T00:14:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary9 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary9 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary10 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary10 snapshot 10G 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary9 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary9 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:14:48.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:48.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary10 10G 2024-09-18T00:14:48.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:48.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:48.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:48.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:48.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:48.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:48.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:14:48.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:14:48.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:14:48.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:14:48.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary10 2024-09-18T00:14:48.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:14:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:14:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:14:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary10 snapshot 2024-09-18T00:14:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:14:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary10 snapshot 2024-09-18T00:14:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:14:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary9/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary9' 2024-09-18T00:14:48.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1017 2024-09-18T00:14:48.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:48.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1021 2024-09-18T00:14:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:14:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:14:48.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:48.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:48.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:14:48.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:48.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:48.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:48.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:14:49.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:49.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:14:49.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:49.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:49.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:49.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:14:49.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1019 2024-09-18T00:14:49.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:49.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1020 2024-09-18T00:14:49.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:14:49.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:49.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary10 2024-09-18T00:14:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary10': 2024-09-18T00:14:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:14:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:14:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:14:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 1ee0c961cf00 2024-09-18T00:14:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.1ee0c961cf00 2024-09-18T00:14:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:14:49.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:14:48 2024 2024-09-18T00:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:14:48 2024 2024-09-18T00:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:14:48 2024 2024-09-18T00:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: a99a3dae-84ba-4485-a01d-736bffc432b5 2024-09-18T00:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:14:49.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:14:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:14:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd9 2024-09-18T00:14:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:14:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd9 2024-09-18T00:14:49.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:14:49.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:14:49.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:14:49.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: f30ceb51-a9cf-44b1-bbab-1451c653d313 2024-09-18T00:14:49.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:14:49.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:14:49.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:49.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:14:49.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:14:50.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:14:50.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:50.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:14:50.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:50.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:50.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:50.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:14:50.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:14:50.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:14:50.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary10 2024-09-18T00:14:50.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd9 test-primary10 2024-09-18T00:14:50.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1023 2024-09-18T00:14:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1024 2024-09-18T00:14:50.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:50.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:14:50.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:50.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:50.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:50.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:14:50.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:50.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:50.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:14:50.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary10 2024-09-18T00:14:50.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:14:50.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary10 2024-09-18T00:14:50.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:14:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:14:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:14:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:14:50.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 63572 2024-09-18T00:14:50.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary10 2024-09-18T00:14:50.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary10 2024-09-18T00:14:50.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary10 2024-09-18T00:14:50.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:14:50.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary10/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary10' 2024-09-18T00:14:51.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:51.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:14:51.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:51.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:51.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:51.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:14:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1025 2024-09-18T00:14:51.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:14:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:14:51.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1028 2024-09-18T00:14:51.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1026 2024-09-18T00:14:52.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1027 2024-09-18T00:14:52.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:52.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:14:52.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:52.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:52.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:52.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:14:52.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:52.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:14:52.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:52.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:52.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:52.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:14:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:14:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:14:53.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:53.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:14:53.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:53.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:53.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:53.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:14:53.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1030 2024-09-18T00:14:53.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1029 2024-09-18T00:14:53.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:53.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:14:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:14:54.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1031 2024-09-18T00:14:54.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:54.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1032 2024-09-18T00:14:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:54.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:54.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:14:54.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:54.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:54.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:54.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:14:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:14:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:14:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:14:55.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:55.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:14:55.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:55.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:55.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:14:55.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:14:55.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1034 2024-09-18T00:14:55.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1033 2024-09-18T00:14:55.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:55.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:14:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:14:56.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:14:56.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1035 2024-09-18T00:14:56.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1036 2024-09-18T00:14:56.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:56.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:14:56.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:56.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:56.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:14:56.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:14:56.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:56.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:57.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1038 2024-09-18T00:14:57.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1037 2024-09-18T00:14:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:57.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:57.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:57.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:14:57.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:57.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:57.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:14:57.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:14:57.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:57.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:14:57.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:57.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:57.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:14:57.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:14:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1039 2024-09-18T00:14:58.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1040 2024-09-18T00:14:58.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:58.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:14:58.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:58.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:58.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:14:58.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:14:58.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:58.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:14:58.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:58.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:58.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:14:58.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:14:58.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:59.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1042 2024-09-18T00:14:59.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:59.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:14:59.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:59.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:59.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:14:59.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:14:59.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:14:59.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:14:59.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:14:59.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:14:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:14:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:14:59.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:14:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1041 2024-09-18T00:14:59.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:00.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:00.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:00.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:00.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:00.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:00.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:00.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:00.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:15:00.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:00.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:00.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:15:00.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:15:00.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1044 2024-09-18T00:15:00.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1043 2024-09-18T00:15:00.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:00.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:15:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:15:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:15:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:15:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:15:01.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:15:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1045 2024-09-18T00:15:01.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:01.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1046 2024-09-18T00:15:01.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:02.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1047 2024-09-18T00:15:02.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:02.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1048 2024-09-18T00:15:02.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:15:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:15:02.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:15:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:15:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:15:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:15:03.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:03.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:15:03.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:03.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:03.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:15:03.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:15:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:15:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:15:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:15:03.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1049 2024-09-18T00:15:03.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1050 2024-09-18T00:15:03.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:03.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1052 2024-09-18T00:15:04.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:04.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:15:04.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:04.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:04.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:15:04.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:15:04.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1051 2024-09-18T00:15:04.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:04.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:04.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:15:04.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:04.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:04.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:15:04.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:15:04.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:05.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:05.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:05.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:05.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:05.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:05.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:05.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:05.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:15:05.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:05.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:05.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:15:05.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:15:05.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1053 2024-09-18T00:15:05.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:05.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1054 2024-09-18T00:15:05.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:06.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1057 2024-09-18T00:15:06.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1056 2024-09-18T00:15:06.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1055 2024-09-18T00:15:06.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:06.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:15:06.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:06.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:06.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:15:06.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:15:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:15:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:15:06.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:15:06.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:06.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:07.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1058 2024-09-18T00:15:07.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:07.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:07.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:15:07.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:07.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:07.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:15:07.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:15:07.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:07.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:15:07.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:07.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:07.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:15:07.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:15:08.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1060 2024-09-18T00:15:08.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1059 2024-09-18T00:15:08.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:08.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:08.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:15:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:15:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:15:08.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:08.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:15:08.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:08.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:08.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:15:08.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:15:09.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1062 2024-09-18T00:15:09.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1061 2024-09-18T00:15:09.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:09.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:09.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:09.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:15:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:15:09.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:15:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:09.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:09.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:15:09.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:09.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:09.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:15:09.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:15:10.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:10.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:15:10.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:10.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:10.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:15:10.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:15:10.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1063 2024-09-18T00:15:10.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1064 2024-09-18T00:15:10.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:10.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:11.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1066 2024-09-18T00:15:11.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1067 2024-09-18T00:15:11.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:15:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:15:11.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:15:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:15:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:11.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:15:11.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:15:11.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1065 2024-09-18T00:15:11.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:12.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1068 2024-09-18T00:15:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:12.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:15:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:15:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:15:12.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:12.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:15:12.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:12.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:12.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:15:12.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:15:13.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1069 2024-09-18T00:15:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:15:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:15:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:15:13.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1070 2024-09-18T00:15:13.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:13.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:15:13.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:13.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:13.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:15:13.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:15:13.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:13.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:14.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1071 2024-09-18T00:15:14.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:14.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:15:14.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:14.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:14.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:15:14.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:15:14.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1072 2024-09-18T00:15:14.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:14.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:14.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:14.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:14.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:14.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:14.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:14.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:14.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:15:14.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:14.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:14.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:15:14.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:15:14.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:15.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:15.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:15:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:15:15.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:15:15.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1074 2024-09-18T00:15:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:15.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1073 2024-09-18T00:15:15.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:16.308 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:15:16.312+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735aedd0 2024-09-18T00:15:16.314 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:15:16.316+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d674563ba0 2024-09-18T00:15:16.323 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:15:16.324+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d68fc696c0 2024-09-18T00:15:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:15:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:15:16.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:15:16.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1075 2024-09-18T00:15:16.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1076 2024-09-18T00:15:16.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1077 2024-09-18T00:15:16.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:16.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:15:16.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:16.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:16.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:15:16.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:15:16.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:16.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:16.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:17.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1078 2024-09-18T00:15:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:17.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:17.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:15:17.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:17.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:17.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:15:17.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:15:17.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:17.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:15:17.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:17.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:17.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:15:17.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:15:18.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:18.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:15:18.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:18.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:18.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:15:18.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:15:18.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1079 2024-09-18T00:15:18.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:18.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:15:18.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:18.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:18.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:15:18.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:15:18.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1080 2024-09-18T00:15:18.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:18.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:19.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1082 2024-09-18T00:15:19.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:19.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:15:19.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:19.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:19.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:15:19.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:15:19.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1081 2024-09-18T00:15:19.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:19.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:19.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:19.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:15:19.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:19.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:19.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:15:19.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:15:19.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:19.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:20.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:20.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:15:20.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:20.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:20.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:15:20.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:15:20.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1084 2024-09-18T00:15:20.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1083 2024-09-18T00:15:20.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:20.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:21.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1085 2024-09-18T00:15:21.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1087 2024-09-18T00:15:21.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:21.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:15:21.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:21.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:21.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:15:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:15:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1086 2024-09-18T00:15:21.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:21.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:21.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:15:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:15:21.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:15:21.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:22.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1088 2024-09-18T00:15:22.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:22.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:22.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:15:22.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:22.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:22.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:15:22.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:15:22.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:22.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:15:22.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:22.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:22.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:15:22.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:15:23.375 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:15:23.376+0000 7fac32fac640 -1 reset not still connected to 0x55a83a447930 2024-09-18T00:15:23.377 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:15:23.380+0000 7fac32fac640 -1 reset not still connected to 0x55a83dd20f70 2024-09-18T00:15:23.378 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:15:23.380+0000 7fac32fac640 -1 reset not still connected to 0x55a851f2ac30 2024-09-18T00:15:23.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1090 2024-09-18T00:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:15:23.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:23.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:15:23.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:23.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:23.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:15:23.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:15:23.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:23.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1089 2024-09-18T00:15:23.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:24.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:24.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:15:24.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:24.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:24.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:15:24.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:15:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:24.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:15:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:15:24.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:15:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1092 2024-09-18T00:15:24.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1091 2024-09-18T00:15:24.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:15:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:25.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:25.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:15:25.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:15:25.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1093 2024-09-18T00:15:25.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:25.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1094 2024-09-18T00:15:25.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:26.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1096 2024-09-18T00:15:26.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1097 2024-09-18T00:15:26.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:26.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:15:26.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:26.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:26.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:15:26.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:15:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1095 2024-09-18T00:15:26.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:26.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:15:26.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:26.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:26.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:15:26.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:15:26.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:26.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:26.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:27.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1098 2024-09-18T00:15:27.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:27.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:27.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:15:27.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:27.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:27.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:15:27.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:15:27.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:27.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:15:27.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:27.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:27.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:15:27.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:15:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:15:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:15:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:15:28.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1099 2024-09-18T00:15:28.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:28.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1100 2024-09-18T00:15:28.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:15:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:15:28.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:15:29.152 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:15:29.156+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735aedd0 2024-09-18T00:15:29.153 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:15:29.156+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d674563ba0 2024-09-18T00:15:29.153 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:15:29.156+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d676dea000 2024-09-18T00:15:29.153 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:15:29.156+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d682a81110 2024-09-18T00:15:29.153 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:15:29.156+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d68fc696c0 2024-09-18T00:15:29.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1101 2024-09-18T00:15:29.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:29.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:15:29.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:29.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:29.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:15:29.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:15:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1102 2024-09-18T00:15:29.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:29.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:15:29.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:29.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:29.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:15:29.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:15:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:29.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:30.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:30.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:15:30.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:30.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:30.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:15:30.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:15:30.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1104 2024-09-18T00:15:30.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:30.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1103 2024-09-18T00:15:30.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:31.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1105 2024-09-18T00:15:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:15:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:15:31.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:15:31.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1106 2024-09-18T00:15:31.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1107 2024-09-18T00:15:31.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:31.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:31.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:15:31.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:31.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:31.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:15:31.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:15:31.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:32.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1108 2024-09-18T00:15:32.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:32.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:32.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:15:32.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:32.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:32.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:15:32.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:15:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:15:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:15:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:15:33.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1109 2024-09-18T00:15:33.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1110 2024-09-18T00:15:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:33.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:33.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:33.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:15:33.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:33.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:33.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:15:33.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:15:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:15:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:15:33.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:15:34.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:34.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:15:34.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:34.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:34.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:15:34.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:15:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:34.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:34.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1112 2024-09-18T00:15:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1111 2024-09-18T00:15:34.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:34.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:15:34.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:34.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:34.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:15:34.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:15:34.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:34.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:35.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:35.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:15:35.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:35.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:35.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:15:35.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:15:35.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1113 2024-09-18T00:15:35.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1114 2024-09-18T00:15:35.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:35.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:36.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:36.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:15:36.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:36.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:36.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:15:36.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:15:36.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:36.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:15:36.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:36.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:36.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:15:36.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:15:36.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1117 2024-09-18T00:15:36.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1115 2024-09-18T00:15:36.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1116 2024-09-18T00:15:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:36.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:36.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:37.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1118 2024-09-18T00:15:37.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:37.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:37.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:15:37.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:37.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:37.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:15:37.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:15:37.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:37.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:15:37.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:37.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:37.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:15:37.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:15:38.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1119 2024-09-18T00:15:38.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:38.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1120 2024-09-18T00:15:38.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:38.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:38.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:15:38.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:38.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:38.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:15:38.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:15:38.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:38.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:15:38.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:38.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:38.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:15:38.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:15:39.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1121 2024-09-18T00:15:39.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:39.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1122 2024-09-18T00:15:39.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:15:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:15:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:15:39.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:39.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:39.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:39.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:39.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:39.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:39.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:39.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:15:39.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:39.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:39.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:15:39.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:15:40.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:40.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:15:40.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:40.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:40.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:15:40.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:15:40.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1123 2024-09-18T00:15:40.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1124 2024-09-18T00:15:40.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:40.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:41.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:41.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:15:41.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:41.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:41.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:15:41.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:15:41.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1126 2024-09-18T00:15:41.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:41.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:15:41.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:41.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:41.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:15:41.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:15:41.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:41.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1125 2024-09-18T00:15:41.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1127 2024-09-18T00:15:41.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:41.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:42.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1128 2024-09-18T00:15:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:15:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:15:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:15:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:15:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:42.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:42.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:15:42.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:15:43.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1129 2024-09-18T00:15:43.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:43.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:43.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:15:43.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:43.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:43.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:15:43.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:15:43.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1130 2024-09-18T00:15:43.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:43.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:15:43.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:43.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:43.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:15:43.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:15:43.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:44.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1132 2024-09-18T00:15:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:15:44.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:44.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:44.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:15:44.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:15:44.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:44.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1131 2024-09-18T00:15:44.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:44.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:44.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:44.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:44.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:44.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:15:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:15:44.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:15:45.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:45.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:15:45.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:45.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:45.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:15:45.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:15:45.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1133 2024-09-18T00:15:45.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1134 2024-09-18T00:15:45.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:45.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:46.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:46.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:15:46.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:46.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:46.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:15:46.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:15:46.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1136 2024-09-18T00:15:46.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:46.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1137 2024-09-18T00:15:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:46.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:15:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:15:46.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:15:46.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1135 2024-09-18T00:15:46.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:46.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:47.286 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:15:47.284+0000 7f849c00d640 -1 reset not still connected to 0x5629ac6572b0 2024-09-18T00:15:47.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1138 2024-09-18T00:15:47.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:15:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:15:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:15:47.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:47.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:15:47.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:47.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:47.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:15:47.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:15:48.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1140 2024-09-18T00:15:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:15:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:15:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:15:48.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:48.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1139 2024-09-18T00:15:48.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:48.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:15:48.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:48.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:48.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:15:48.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:15:48.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:49.221 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-18T00:15:49.215+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0ca37380 2024-09-18T00:15:49.221 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-18T00:15:49.215+0000 7fbbe8b19640 -1 reset not still connected to 0x561a1cf284e0 2024-09-18T00:15:49.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1141 2024-09-18T00:15:49.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:49.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:49.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:49.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:49.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:49.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:49.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1142 2024-09-18T00:15:49.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:49.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:49.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:15:49.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:49.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:49.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:15:49.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:15:49.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:49.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:49.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:15:49.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:49.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:49.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:15:49.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:15:50.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:50.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:15:50.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:50.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:50.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:15:50.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:15:50.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1143 2024-09-18T00:15:50.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:50.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1144 2024-09-18T00:15:50.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:51.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1146 2024-09-18T00:15:51.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1145 2024-09-18T00:15:51.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1147 2024-09-18T00:15:51.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:15:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:15:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:15:51.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:51.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:51.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:51.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:51.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:15:51.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:51.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:51.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:15:51.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:15:52.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1148 2024-09-18T00:15:52.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:15:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:15:52.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:15:52.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:52.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:15:52.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:52.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:52.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:15:52.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:15:53.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1149 2024-09-18T00:15:53.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:53.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:15:53.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:53.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:53.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:15:53.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:15:53.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:53.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1150 2024-09-18T00:15:53.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:53.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:53.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:15:53.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:53.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:53.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:15:53.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:15:54.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:15:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:15:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:15:54.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:54.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:54.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:54.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:54.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:54.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:54.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:54.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:15:54.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:54.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:54.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:15:54.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:15:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1152 2024-09-18T00:15:54.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1151 2024-09-18T00:15:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:54.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:55.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:55.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:15:55.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:55.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:55.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:15:55.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:15:55.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1153 2024-09-18T00:15:55.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:55.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1154 2024-09-18T00:15:55.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:56.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:56.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:15:56.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:56.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:56.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:15:56.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:15:56.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1155 2024-09-18T00:15:56.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:56.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:15:56.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:56.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:56.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:15:56.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:15:56.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1157 2024-09-18T00:15:56.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1156 2024-09-18T00:15:56.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:56.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:56.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:57.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1158 2024-09-18T00:15:57.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:57.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:57.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:15:57.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:57.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:57.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:15:57.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:15:57.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:57.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:15:57.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:57.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:57.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:15:57.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:15:58.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1160 2024-09-18T00:15:58.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:58.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:15:58.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:58.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:58.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:15:58.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:15:58.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1159 2024-09-18T00:15:58.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:58.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:15:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:58.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:15:58.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:15:59.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1162 2024-09-18T00:15:59.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1161 2024-09-18T00:15:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:15:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:15:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:15:59.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:15:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:15:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:15:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:15:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:15:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:15:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:15:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:15:59.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:15:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:15:59.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:00.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:00.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:16:00.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:00.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:00.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:16:00.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:16:00.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1164 2024-09-18T00:16:00.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1163 2024-09-18T00:16:00.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:00.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:01.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:01.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:16:01.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:01.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:01.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:16:01.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:16:01.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1165 2024-09-18T00:16:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1167 2024-09-18T00:16:01.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:16:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:16:01.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:16:01.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1166 2024-09-18T00:16:01.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:01.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:01.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:02.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1168 2024-09-18T00:16:02.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:02.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:02.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:16:02.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:02.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:02.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:16:02.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:16:02.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:02.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:16:02.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:02.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:02.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:16:02.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:16:03.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1169 2024-09-18T00:16:03.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1170 2024-09-18T00:16:03.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:03.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:16:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:16:03.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:16:03.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:03.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:16:03.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:03.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:03.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:16:03.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:16:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:16:04.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:04.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:04.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:16:04.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:16:04.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1171 2024-09-18T00:16:04.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:04.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:16:04.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:04.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:04.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:16:04.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:16:04.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1172 2024-09-18T00:16:04.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:04.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:04.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:16:04.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:04.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:04.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:16:04.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:16:04.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:05.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:05.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:16:05.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:05.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:05.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:16:05.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:16:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1174 2024-09-18T00:16:05.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:05.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1173 2024-09-18T00:16:05.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:06.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:06.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:16:06.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:06.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:06.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:16:06.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:16:06.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:16:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:16:06.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:16:06.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1175 2024-09-18T00:16:06.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1177 2024-09-18T00:16:06.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1176 2024-09-18T00:16:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:06.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:07.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1178 2024-09-18T00:16:07.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:07.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:16:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:16:07.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:16:07.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:07.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:16:07.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:07.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:07.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:16:07.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:16:08.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1180 2024-09-18T00:16:08.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1179 2024-09-18T00:16:08.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:08.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:08.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:08.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:16:08.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:08.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:08.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:16:08.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:16:08.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:08.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:16:08.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:08.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:08.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:16:08.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:16:09.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1181 2024-09-18T00:16:09.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:09.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1182 2024-09-18T00:16:09.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:09.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:09.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:16:09.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:09.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:09.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:16:09.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:16:09.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:09.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:16:09.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:09.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:09.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:16:09.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:16:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:16:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:16:09.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:16:10.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1183 2024-09-18T00:16:10.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:10.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:16:10.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:16:10.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:16:10.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1184 2024-09-18T00:16:10.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:16:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:16:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:16:11.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:11.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:16:11.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:11.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:11.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:16:11.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:16:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1185 2024-09-18T00:16:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1186 2024-09-18T00:16:11.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1187 2024-09-18T00:16:11.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:11.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:11.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:16:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:16:12.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:16:12.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1188 2024-09-18T00:16:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:12.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:12.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:16:12.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:12.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:12.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:16:12.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:16:13.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:13.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:16:13.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:13.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:13.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:16:13.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:16:13.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1190 2024-09-18T00:16:13.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:13.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:16:13.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:13.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:13.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:16:13.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:16:13.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1189 2024-09-18T00:16:13.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:13.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:14.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:14.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:16:14.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:14.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:14.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:16:14.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:16:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:16:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:16:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:16:14.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1191 2024-09-18T00:16:14.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:14.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:16:14.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:14.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:14.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:16:14.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:16:14.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1192 2024-09-18T00:16:14.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:14.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:15.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:15.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:16:15.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:15.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:15.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:16:15.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:16:15.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1193 2024-09-18T00:16:15.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1194 2024-09-18T00:16:15.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:15.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:16.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:16.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:16:16.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:16.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:16.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:16:16.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:16:16.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1197 2024-09-18T00:16:16.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1195 2024-09-18T00:16:16.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1196 2024-09-18T00:16:16.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:16.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:16:16.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:16.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:16.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:16:16.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:16:16.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:16.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:16.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:17.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1198 2024-09-18T00:16:17.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:16:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:17.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:16:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:16:17.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:16:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:16:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:16:18.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1199 2024-09-18T00:16:18.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:18.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1200 2024-09-18T00:16:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:16:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:16:18.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:16:18.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:16:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:16:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:16:19.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1201 2024-09-18T00:16:19.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1202 2024-09-18T00:16:19.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:19.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:16:19.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:19.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:19.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:16:19.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:16:19.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:19.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:16:19.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:19.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:19.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:16:19.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:16:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:16:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:19.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:16:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:16:20.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:20.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:16:20.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:20.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:20.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:16:20.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:16:20.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1203 2024-09-18T00:16:20.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1204 2024-09-18T00:16:20.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:20.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:21.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:16:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:16:21.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:16:21.376 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:16:21.379+0000 7fac32fac640 -1 reset not still connected to 0x55a84f7e01a0 2024-09-18T00:16:21.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1207 2024-09-18T00:16:21.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1205 2024-09-18T00:16:21.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:21.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:16:21.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:21.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:21.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:16:21.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:16:21.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:21.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1206 2024-09-18T00:16:21.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:21.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:22.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1208 2024-09-18T00:16:22.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:22.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:22.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:16:22.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:22.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:22.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:16:22.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:16:22.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:22.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:16:22.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:22.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:22.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:16:22.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:16:23.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1209 2024-09-18T00:16:23.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1210 2024-09-18T00:16:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:16:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:16:23.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:16:23.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:23.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:23.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:16:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:16:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:16:24.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1211 2024-09-18T00:16:24.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1212 2024-09-18T00:16:24.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:24.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:24.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:16:24.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:24.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:24.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:16:24.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:16:24.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:24.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:24.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:16:24.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:24.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:24.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:16:24.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:16:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:16:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:16:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:16:25.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:25.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:16:25.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:25.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:25.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:16:25.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:16:25.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1213 2024-09-18T00:16:25.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:25.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1214 2024-09-18T00:16:25.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:26.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1217 2024-09-18T00:16:26.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:26.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:16:26.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:26.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:26.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:16:26.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:16:26.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:26.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:16:26.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:26.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:26.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:16:26.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:16:26.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1215 2024-09-18T00:16:26.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1216 2024-09-18T00:16:26.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:26.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:26.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:27.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1218 2024-09-18T00:16:27.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:27.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:27.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:16:27.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:27.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:27.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:16:27.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:16:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:16:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:16:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:16:28.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1219 2024-09-18T00:16:28.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1220 2024-09-18T00:16:28.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:28.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:16:28.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:28.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:28.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:16:28.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:16:28.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:28.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:28.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:28.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:16:28.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:28.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:28.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:16:28.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:16:29.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1222 2024-09-18T00:16:29.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:29.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:16:29.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:29.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:29.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:16:29.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:16:29.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:29.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:16:29.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:29.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:29.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:16:29.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:16:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1221 2024-09-18T00:16:29.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:29.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:16:29.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:29.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:29.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:16:29.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:16:29.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:29.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:16:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:16:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:16:30.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1223 2024-09-18T00:16:30.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:30.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1224 2024-09-18T00:16:30.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:31.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1226 2024-09-18T00:16:31.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1225 2024-09-18T00:16:31.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1227 2024-09-18T00:16:31.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:16:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:16:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:16:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:16:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:31.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:16:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:16:31.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:32.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1228 2024-09-18T00:16:32.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:16:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:16:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:16:32.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:32.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:16:32.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:32.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:32.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:16:32.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:16:33.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1229 2024-09-18T00:16:33.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1230 2024-09-18T00:16:33.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:33.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:16:33.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:33.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:33.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:16:33.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:16:33.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:33.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:33.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:33.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:16:33.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:33.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:33.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:16:33.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:16:34.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:34.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:16:34.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:34.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:34.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:16:34.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:16:34.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1231 2024-09-18T00:16:34.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:34.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:16:34.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:34.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:34.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:16:34.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:16:34.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:34.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:16:34.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:34.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:34.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:16:34.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:16:34.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1232 2024-09-18T00:16:34.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:34.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:35.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:35.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:16:35.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:35.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:35.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:16:35.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:16:35.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1234 2024-09-18T00:16:35.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1233 2024-09-18T00:16:35.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:35.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:16:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:16:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:16:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:16:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:16:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:16:36.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1236 2024-09-18T00:16:36.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1235 2024-09-18T00:16:36.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:36.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:36.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1237 2024-09-18T00:16:36.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:37.026 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:16:37.025+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T00:16:37.027 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:16:37.025+0000 7fd4c158b640 -1 reset not still connected to 0x56435b09f380 2024-09-18T00:16:37.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1238 2024-09-18T00:16:37.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:37.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:16:37.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:37.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:37.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:16:37.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:16:37.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:37.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:16:37.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:37.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:37.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:16:37.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:16:38.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1240 2024-09-18T00:16:38.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1239 2024-09-18T00:16:38.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:38.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:38.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:38.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:16:38.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:38.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:38.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:16:38.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:16:38.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:38.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:16:38.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:38.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:38.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:16:38.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:16:39.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1241 2024-09-18T00:16:39.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:39.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1242 2024-09-18T00:16:39.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:39.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:39.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:16:39.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:39.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:39.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:16:39.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:16:39.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:39.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:16:39.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:39.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:39.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:16:39.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:16:39.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:39.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:16:39.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:39.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:39.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:16:39.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:16:40.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1243 2024-09-18T00:16:40.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:40.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:40.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:16:40.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:40.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:40.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:16:40.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:16:40.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1244 2024-09-18T00:16:40.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:16:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:16:41.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:16:41.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:41.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:16:41.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:41.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:41.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:16:41.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:16:41.323 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:16:41.327+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d68fc696c0 2024-09-18T00:16:41.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1247 2024-09-18T00:16:41.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1246 2024-09-18T00:16:41.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1245 2024-09-18T00:16:41.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:41.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:42.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1248 2024-09-18T00:16:42.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:42.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:42.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:16:42.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:42.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:42.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:16:42.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:16:42.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:42.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:16:42.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:42.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:42.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:16:42.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:16:43.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:43.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:16:43.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:43.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:43.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:16:43.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:16:43.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1250 2024-09-18T00:16:43.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1249 2024-09-18T00:16:43.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:43.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:16:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:16:43.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:16:44.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1252 2024-09-18T00:16:44.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:44.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:16:44.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:44.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:44.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:16:44.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:16:44.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:44.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:16:44.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:44.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:16:44.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:16:44.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1251 2024-09-18T00:16:44.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:44.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:16:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:16:44.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:16:44.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:45.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:45.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:16:45.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:45.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:45.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:16:45.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:16:45.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1254 2024-09-18T00:16:45.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1253 2024-09-18T00:16:45.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:45.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:45.416 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:16:45.418+0000 7fac32fac640 -1 reset not still connected to 0x55a83dd20f70 2024-09-18T00:16:45.416 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:16:45.418+0000 7fac32fac640 -1 reset not still connected to 0x55a851f2ac30 2024-09-18T00:16:46.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1255 2024-09-18T00:16:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:16:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:16:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:16:46.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:46.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:16:46.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:46.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:46.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:16:46.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:16:46.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1256 2024-09-18T00:16:46.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1257 2024-09-18T00:16:46.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:46.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:47.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1258 2024-09-18T00:16:47.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:47.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:47.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:16:47.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:47.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:47.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:16:47.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:16:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:47.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:16:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:16:47.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:16:48.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1260 2024-09-18T00:16:48.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:48.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:48.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:16:48.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:48.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:48.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:16:48.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:16:48.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:48.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:16:48.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:48.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:48.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:16:48.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:16:48.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1259 2024-09-18T00:16:48.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:49.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1261 2024-09-18T00:16:49.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:49.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:16:49.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:49.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:49.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:16:49.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:16:49.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:49.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:16:49.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:49.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:49.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:16:49.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:16:49.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:49.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1262 2024-09-18T00:16:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:16:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:16:49.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:16:49.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:16:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:16:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:16:50.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1264 2024-09-18T00:16:50.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:50.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1263 2024-09-18T00:16:50.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:51.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:51.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:16:51.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:51.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:51.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:16:51.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:16:51.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1265 2024-09-18T00:16:51.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1266 2024-09-18T00:16:51.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:51.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:16:51.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:51.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:51.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:16:51.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:16:51.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:51.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:51.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1267 2024-09-18T00:16:51.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:51.462 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:16:51.466+0000 7f849c00d640 -1 reset not still connected to 0x5629b0727450 2024-09-18T00:16:52.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1268 2024-09-18T00:16:52.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:52.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:52.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:16:52.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:52.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:52.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:16:52.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:16:52.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:16:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:16:52.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:16:53.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1270 2024-09-18T00:16:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:16:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:16:53.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:16:53.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1269 2024-09-18T00:16:53.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:53.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:53.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:53.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:16:53.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:53.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:53.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:16:53.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:16:54.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1271 2024-09-18T00:16:54.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:54.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:16:54.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:54.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:54.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:16:54.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:16:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1272 2024-09-18T00:16:54.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:54.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:16:54.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:54.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:54.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:16:54.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:16:54.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:54.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:54.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:16:54.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:16:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:16:54.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:55.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:55.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:16:55.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:55.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:55.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:16:55.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:16:55.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1274 2024-09-18T00:16:55.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1273 2024-09-18T00:16:55.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:55.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:56.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1275 2024-09-18T00:16:56.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1277 2024-09-18T00:16:56.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1276 2024-09-18T00:16:56.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:16:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:56.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:16:56.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:16:56.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:56.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:56.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:16:56.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:56.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:56.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:16:56.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:16:56.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:57.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1278 2024-09-18T00:16:57.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:57.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:57.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:16:57.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:57.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:57.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:16:57.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:16:57.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:57.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:16:57.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:57.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:57.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:16:57.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:16:58.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:58.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:16:58.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:58.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:58.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:16:58.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:16:58.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1280 2024-09-18T00:16:58.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1279 2024-09-18T00:16:58.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:58.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:16:58.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:16:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:16:58.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:58.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:59.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:16:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:16:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:16:59.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1281 2024-09-18T00:16:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:16:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:59.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:16:59.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:16:59.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:16:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:16:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:16:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:16:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:16:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:16:59.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1282 2024-09-18T00:16:59.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:16:59.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:17:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:17:00.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:17:00.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1283 2024-09-18T00:17:00.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1284 2024-09-18T00:17:00.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:00.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:01.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1285 2024-09-18T00:17:01.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1288 2024-09-18T00:17:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:01.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:01.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:17:01.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:17:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:17:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:17:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:17:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:17:01.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1287 2024-09-18T00:17:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:01.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:02.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1289 2024-09-18T00:17:02.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:02.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:02.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:17:02.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:02.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:02.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:17:02.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:17:02.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:17:02.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 63702 2024-09-18T00:17:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1291 2024-09-18T00:17:03.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1290 2024-09-18T00:17:03.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:03.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:17:03.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 63838 2024-09-18T00:17:03.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:03.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:17:03.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:03.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:03.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:17:03.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:17:04.125 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:17:04.124+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b040 2024-09-18T00:17:04.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:04.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:17:04.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:04.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:04.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:17:04.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:17:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1292 2024-09-18T00:17:04.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:04.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:04.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:17:04.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:04.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:04.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:17:04.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:17:04.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:17:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:17:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:17:05.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:17:05.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 64017 2024-09-18T00:17:05.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1293 2024-09-18T00:17:05.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:06.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:17:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:17:06.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:17:06.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:17:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:17:06.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1295 2024-09-18T00:17:06.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1296 2024-09-18T00:17:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1294 2024-09-18T00:17:06.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:06.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:06.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:07.326 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:17:07.329+0000 7fac32fac640 -1 reset not still connected to 0x55a851f2ac30 2024-09-18T00:17:07.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:17:07.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 64242 2024-09-18T00:17:08.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1297 2024-09-18T00:17:08.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1298 2024-09-18T00:17:08.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:08.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:08.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:08.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:17:08.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:08.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:08.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:17:08.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:17:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:09.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:17:09.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:09.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:09.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:17:09.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:17:09.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:09.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:17:09.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:17:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:17:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:17:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 64397 2024-09-18T00:17:10.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1299 2024-09-18T00:17:10.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:11.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1301 2024-09-18T00:17:11.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1300 2024-09-18T00:17:11.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:17:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 64632 2024-09-18T00:17:11.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:11.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:17:11.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:11.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:11.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:17:11.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:17:11.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:11.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:13.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1302 2024-09-18T00:17:13.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:13.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:17:13.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 64799 2024-09-18T00:17:14.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:17:14.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 65042 2024-09-18T00:17:14.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:17:14.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:17:14.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:17:14.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:17:14.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:17:14.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:17:16.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1303 2024-09-18T00:17:16.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:17:16.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 65299 2024-09-18T00:17:16.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:17:19.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:17:19.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:17:19.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 63573 2024-09-18T00:19:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:19:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:19:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:19:35.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 63703 2024-09-18T00:19:35.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:19:35.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:19:35.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:19:35.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 63839 2024-09-18T00:19:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:19:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:19:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:19:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 64018 2024-09-18T00:19:38.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:19:38.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:19:38.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:19:38.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 64243 2024-09-18T00:19:40.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:19:40.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:19:40.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:19:40.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 64398 2024-09-18T00:19:42.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:19:42.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:19:42.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:19:42.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 64633 2024-09-18T00:19:44.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:19:44.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:19:44.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:19:44.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 64800 2024-09-18T00:19:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:19:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:19:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:19:46.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 65043 2024-09-18T00:19:48.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:19:48.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:19:48.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:19:48.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 65300 2024-09-18T00:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:19:50.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:19:50.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:19:50.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 1 2024-09-18T00:19:50.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:19:50.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd0 2024-09-18T00:19:50.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary1 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary1 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary1 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 2 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd1 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary2 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary2 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 3 2024-09-18T00:19:50.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary2 2024-09-18T00:19:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd2 2024-09-18T00:19:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:19:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 4 2024-09-18T00:19:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary3 2024-09-18T00:19:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary3 2024-09-18T00:19:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:19:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:19:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary3 2024-09-18T00:19:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd3 2024-09-18T00:19:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary4 2024-09-18T00:19:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary4 2024-09-18T00:19:50.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:19:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary4 2024-09-18T00:19:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:19:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:19:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 5 2024-09-18T00:19:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd4 2024-09-18T00:19:50.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary5 2024-09-18T00:19:50.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary5 2024-09-18T00:19:50.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:19:50.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:19:50.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary5 2024-09-18T00:19:50.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:19:50.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 6 2024-09-18T00:19:50.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd5 2024-09-18T00:19:50.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary6 2024-09-18T00:19:50.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:19:50.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary6 2024-09-18T00:19:50.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:19:50.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary6 2024-09-18T00:19:50.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 7 2024-09-18T00:19:50.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd6 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary7 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary7 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary7 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 71140 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 10 2024-09-18T00:19:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd9 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary10 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary10 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary10 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 9 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd8 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary9 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary9 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary9 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 8 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd7 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary8 2024-09-18T00:19:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary8 2024-09-18T00:19:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:19:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary8 2024-09-18T00:19:51.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:19:51.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:19:51.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:19:51.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:19:51.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:19:51.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:19:51.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:19:51.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:19:51.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:19:51.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:19:51.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:19:51.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:19:51.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:19:51.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:19:51.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:19:51.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:19:51.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:19:51.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:19:51.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:19:51.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:20:49.165 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:20:49.161+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b040 2024-09-18T00:20:50.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=d58f036957c1e0f3abdd0c642f8a89bd 2024-09-18T00:20:50.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary6 2024-09-18T00:20:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary6 2024-09-18T00:20:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:20:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary6 2024-09-18T00:20:52.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=c9a7d8902d05cc5151a7e74576ef32de 2024-09-18T00:20:52.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary3 2024-09-18T00:20:52.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=65dcc38740816803f8112679a1bdff56 2024-09-18T00:20:52.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary2 2024-09-18T00:20:52.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=dc9d4c9ae53d7213d676da0b95a097a3 2024-09-18T00:20:52.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary8 2024-09-18T00:20:53.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=bf248b7b84196905c9cb30e67a56b6ee 2024-09-18T00:20:53.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary4 2024-09-18T00:20:53.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=966300c16ad66f152c865ef9433a8c78 2024-09-18T00:20:53.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary9 2024-09-18T00:20:53.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary3 2024-09-18T00:20:53.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:53.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:53.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:20:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary3 2024-09-18T00:20:53.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=9a73956cf5e56c57066849d632e1cad7 2024-09-18T00:20:53.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary7 2024-09-18T00:20:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=f3b60fabbd942f607ea139df31608403 2024-09-18T00:20:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary5 2024-09-18T00:20:53.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary2 2024-09-18T00:20:53.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:53.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:53.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:20:53.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary2 2024-09-18T00:20:54.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=ca5c43a28591836e4aa15c06986aee5a 2024-09-18T00:20:54.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary10 2024-09-18T00:20:54.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary9 2024-09-18T00:20:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:20:54.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary9 2024-09-18T00:20:54.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary8 2024-09-18T00:20:54.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:54.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:54.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:20:54.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary8 2024-09-18T00:20:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:20:54.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+unknown 2024-09-18T00:20:54.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:54.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:54.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:20:54.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:54.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:54.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:54.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:54.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:20:54.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary4 2024-09-18T00:20:54.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:54.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:54.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:20:54.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary4 2024-09-18T00:20:54.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary7 2024-09-18T00:20:54.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:54.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:54.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:20:54.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary7 2024-09-18T00:20:54.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary5 2024-09-18T00:20:54.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:54.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:54.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:20:54.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary5 2024-09-18T00:20:54.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=7ebd324896096b75aaa5dacbc10e0196 2024-09-18T00:20:54.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary1 2024-09-18T00:20:54.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary10 2024-09-18T00:20:54.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:54.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:54.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:20:54.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary10 2024-09-18T00:20:55.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:20:55.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+unknown 2024-09-18T00:20:55.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:55.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:55.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:20:55.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:55.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:55.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:55.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:55.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:20:55.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:20:55.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+unknown 2024-09-18T00:20:55.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:55.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:55.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:20:55.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:55.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:55.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:55.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:55.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:20:55.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T00:20:55.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:55.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:55.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:20:55.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:55.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:55.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:55.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:55.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:55.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:20:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary1 2024-09-18T00:20:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:20:55.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary1 2024-09-18T00:20:55.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:20:55.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 88c5b661-6663-4bbd-986c-9e5a7fb24308 2024-09-18T00:20:55.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:55.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:20:55.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:55.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54 2024-09-18T00:20:55.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:55.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:55.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:55.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:20:55.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:55' 2024-09-18T00:20:55.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:55.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:20:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:20:55.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:20:55.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:20:55.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:20:55.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:20:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:20:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:20:55.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:20:55.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:20:55.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:20:56.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:54' 2024-09-18T00:20:56.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+unknown '2024-09-18 00:20:54' 2024-09-18T00:20:56.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:56.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:20:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:54' 2024-09-18T00:20:56.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:56.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:20:56.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T00:20:56.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:56.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:56.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:20:56.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:56.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:56.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:56.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:56.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:56.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:20:56.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:20:56.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:20:56.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:20:56.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:20:56.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:56.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:56.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:20:56.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:56.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:56.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:56.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:56.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:56.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:20:56.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+unknown 2024-09-18T00:20:56.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:56.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:56.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:20:56.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:56.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:56.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:56.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:56.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:20:56.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+unknown 2024-09-18T00:20:56.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:20:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:20:56.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:20:56.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:20:56.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:20:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":907264.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618618,"remote_snapshot_timestamp":1726618618,"replay_state":"idle"}2024-09-18 00:20:39image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:20:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2613248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618635,"remote_snapshot_timestamp":1726618635,"replay_state":"idle"}2024-09-18 00:20:39image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:20:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:20:56image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:20:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:20:56image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:20:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2637824.0,"last_snapshot_bytes":5132288,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618623,"remote_snapshot_timestamp":1726618623,"replay_state":"idle"}2024-09-18 00:20:39image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:20:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2168832.0,"last_snapshot_bytes":2641920,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618625,"remote_snapshot_timestamp":1726618625,"replay_state":"idle"}2024-09-18 00:20:39image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:20:55image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:20:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2607104.0,"last_snapshot_bytes":221184,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618629,"remote_snapshot_timestamp":1726618629,"replay_state":"idle"}2024-09-18 00:20:39image-primary859ac8874-602c-4836-b596-7e605198b3a1up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:20:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1751040.0,"last_snapshot_bytes":1769472,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618630,"remote_snapshot_timestamp":1726618630,"replay_state":"idle"}2024-09-18 00:20:39image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:20:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2140160.0,"last_snapshot_bytes":2646016,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618632,"remote_snapshot_timestamp":1726618632,"replay_state":"idle"}2024-09-18 00:20:39' 2024-09-18T00:20:56.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:20:56.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+unknown 2024-09-18T00:20:56.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:56.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:56.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:20:56.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:56.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:56.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:56.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:56.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:20:56.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:20:56.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:20:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+unknown 2024-09-18T00:20:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:20:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:56.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:20:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+unknown 2024-09-18T00:20:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:20:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:20:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+unknown 2024-09-18T00:20:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:20:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:20:56.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:20:56.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 74d7ffd9-fbb0-4520-9d48-784c6cd85819 2024-09-18T00:20:56.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54 2024-09-18T00:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:56' 2024-09-18T00:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:56.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:20:56.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:56.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:20:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:20:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4 2024-09-18T00:20:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:20:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:20:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54 2024-09-18T00:20:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:20:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:56' 2024-09-18T00:20:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:20:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:20:56.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:54' 2024-09-18T00:20:56.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:20:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:20:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:20:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:20:56.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:20:56.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:20:56.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:20:56.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary6 up+unknown 2024-09-18T00:20:56.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:20:56.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:56.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:20:56.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:56.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:56.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:56.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:56.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:20:57.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:20:57.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+unknown 2024-09-18T00:20:57.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:57.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:57.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:20:57.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:57.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:57.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:57.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:57.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:20:57.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:20:57.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:57.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:57.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:20:57.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:57.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:57.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:57.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:57.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:57.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:20:57.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T00:20:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:20:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:20:57.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T00:20:57.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:57.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:57.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:20:57.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:57.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:57.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:57.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:57.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:57.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:20:57.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:20:57.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:20:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:20:57.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:20:57.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:57.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:57.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:20:57.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:57.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:57.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:57.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:57.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:57.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:20:57.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:20:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:20:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:20:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:20:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 59ac8874-602c-4836-b596-7e605198b3a1 2024-09-18T00:20:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:20:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:20:57.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:57.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54 2024-09-18T00:20:57.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:57.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:57.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:57.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:20:57.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:57' 2024-09-18T00:20:57.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:57.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:20:57.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:57.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:20:57.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a99a3dae-84ba-4485-a01d-736bffc432b5 2024-09-18T00:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54 2024-09-18T00:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:57' 2024-09-18T00:20:57.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:57.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:20:57.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:57.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:20:57.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:20:57.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 0c2b99dd-43bc-4302-bae9-69b6a68f20e6 2024-09-18T00:20:57.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:20:57.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:20:57.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54 2024-09-18T00:20:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:20:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:57' 2024-09-18T00:20:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:20:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:20:57.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:20:57.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 158b22b4-54b8-411f-9e7e-2163182393ca 2024-09-18T00:20:57.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:20:57.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:20:57.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:57.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54 2024-09-18T00:20:57.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:57.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:57.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:20:57.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2637824.0,"last_snapshot_bytes":5132288,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618623,"remote_snapshot_timestamp":1726618623,"replay_state":"idle"} 2024-09-18T00:20:57.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:39' 2024-09-18T00:20:57.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:57.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:20:57.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:57.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:20:57.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:20:57.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8d545589-7b5b-44e3-b6e8-4df0695358a6 2024-09-18T00:20:57.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:20:57.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:20:57.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:57.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54 2024-09-18T00:20:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:20:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:57' 2024-09-18T00:20:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:20:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:57.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:20:57.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:20:57.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 79b8265e-910e-4b6e-9e5c-df55997a9d65 2024-09-18T00:20:57.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:20:57.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:20:57.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:57.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54 2024-09-18T00:20:57.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:57.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:57.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:57.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:20:57.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:57' 2024-09-18T00:20:57.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:57.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:20:57.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:57.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:20:57.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary6 up+unknown '' '' 2024-09-18T00:20:57.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:20:57.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:57.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:20:57.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:57.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:57.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:57.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:57.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:57.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary6 2024-09-18T00:20:57.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:20:57.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 88c5b661-6663-4bbd-986c-9e5a7fb24308 2024-09-18T00:20:57.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:57.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:20:57.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:57.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:55 2024-09-18T00:20:57.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:57.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:20:57.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:57.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:20:57.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54' 2024-09-18T00:20:57.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:57.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:20:57.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:20:57.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:20:57.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:20:57.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:20:57.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:20:57.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:20:57.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:20:57.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:20:57.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:20:57.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:20:57.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:55' 2024-09-18T00:20:57.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary6 up+unknown '2024-09-18 00:20:55' 2024-09-18T00:20:57.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:20:57.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:57.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:20:57.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:57.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:55' 2024-09-18T00:20:57.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:57.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:57.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:20:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":907264.0,"last_snapshot_bytes":180224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618618,"remote_snapshot_timestamp":1726618618,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:20:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:20:54image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:20:57cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:20:54image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:20:56cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:20:54image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:20:56cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:20:54image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2637824.0,"last_snapshot_bytes":5132288,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618623,"remote_snapshot_timestamp":1726618623,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:20:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:20:54image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:20:57cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:20:54image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:20:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:54image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:20:57cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:20:54image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:20:57cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:20:54image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:20:57cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:20:54' 2024-09-18T00:20:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:20:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:20:58.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:55' 2024-09-18T00:20:58.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:20:58.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:20:58.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:20:58.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:20:58.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:20:58.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:20:58.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:20:58.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary6 2024-09-18T00:20:58.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:20:58.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:58.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:20:58.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:20:58.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary6 2024-09-18T00:20:58.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:20:58.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:58.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:20:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:58.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:58.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:20:58.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:20:58.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 684b85b9-e82c-49e4-af6b-eb72455a5948 2024-09-18T00:20:58.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:20:58.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:20:58.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:58.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54 2024-09-18T00:20:58.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:58.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:58.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:58.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:20:58.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:58' 2024-09-18T00:20:58.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:58.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:20:58.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:58.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:20:58.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T00:20:58.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:58.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:58.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:20:58.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:58.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:58.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:58.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:58.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:58.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:20:58.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:20:58.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:58.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:58.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:20:58.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:58.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:58.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:58.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:58.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:58.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:20:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:20:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 74d7ffd9-fbb0-4520-9d48-784c6cd85819 2024-09-18T00:20:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:20:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:20:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54 2024-09-18T00:20:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:58.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:58.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:20:58.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:56' 2024-09-18T00:20:58.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:58.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:20:58.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:58.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:20:58.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:20:58.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4 2024-09-18T00:20:58.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:20:58.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:20:58.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:58.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:54 2024-09-18T00:20:58.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:58.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:58.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:58.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:20:58.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:56' 2024-09-18T00:20:58.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:58.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:20:58.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:58.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:20:59.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:20:59.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:20:59.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:20:59.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd0 2024-09-18T00:20:59.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:20:59.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:20:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:20:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:20:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:59.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:20:59.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T00:20:59.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:59.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:59.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:20:59.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:59.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:59.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:59.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:59.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:59.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:20:59.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T00:20:59.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:59.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:59.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:20:59.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:59.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:59.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:59.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:59.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:59.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:20:59.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:20:59.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:59.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:59.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:20:59.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:59.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:59.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:59.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:59.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:59.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:20:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:20:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:20:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:59.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:59.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:59.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:59.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:59.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:20:59.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:20:59.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:59.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:59.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:20:59.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:59.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:59.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:59.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:59.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:59.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:20:59.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:20:59.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 59ac8874-602c-4836-b596-7e605198b3a1 2024-09-18T00:20:59.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:59.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:20:59.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:59.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:20:59.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:59.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:59.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:59.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:20:59.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:20:59.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:59.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:20:59.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:20:59.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:20:59.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:20:59.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:20:59.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:20:59.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:20:59.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:20:59.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:20:59.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:20:59.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:20:59.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:20:59.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:20:59.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:59.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:59.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:20:59.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:59.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:20:59.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:59.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:59.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:20:59.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:20:59.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a99a3dae-84ba-4485-a01d-736bffc432b5 2024-09-18T00:20:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:20:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:20:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:20:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:20:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:20:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:20:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:20:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 0c2b99dd-43bc-4302-bae9-69b6a68f20e6 2024-09-18T00:20:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:20:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:20:59.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:59.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:59.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:59.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:20:59.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:20:59.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:59.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:20:59.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:20:59.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:20:59.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:20:59.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:20:59.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:20:59.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:20:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:20:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:20:59.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:20:59.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:20:59.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:20:59.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:20:59.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:20:59.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:20:59.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:20:59.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:20:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:20:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:20:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:20:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:20:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:20:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:20:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:20:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:20:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:20:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:59.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:20:59.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:59.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:20:59.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:59.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:59.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:20:59.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:20:59.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 158b22b4-54b8-411f-9e7e-2163182393ca 2024-09-18T00:20:59.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:59.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:20:59.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:59.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:20:59.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:59.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:59.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:59.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:20:59.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:20:59.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:59.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:20:59.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:20:59.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:20:59.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:20:59.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:20:59.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:20:59.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:20:59.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:20:59.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:20:59.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:20:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:20:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:20:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:20:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:20:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:20:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:59.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:20:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:20:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 79b8265e-910e-4b6e-9e5c-df55997a9d65 2024-09-18T00:20:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:20:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:20:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:20:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:20:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:20:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:20:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:20:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:20:59.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:20:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:20:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:20:59.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:20:59.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:20:59.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:20:59.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:20:59.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:20:59.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:20:59.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:20:59.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:20:59.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:20:59.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:20:59.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:20:59.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:20:59.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:59.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:20:59.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:59.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:20:59.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:20:59.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:20:59.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:20:59.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:20:59.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:20:59.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:20:59.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:20:59.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:20:59.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:20:59.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:20:59.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:20:59.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary10 up+unknown 2024-09-18T00:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:20:59.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:21:00.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:00.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:00.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:21:00.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:00.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:21:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:21:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary4 up+unknown 2024-09-18T00:21:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:21:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:21:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:21:00.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:00.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:21:00.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:00.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:00.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:00.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:00.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:00.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:21:00.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:00.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:00.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:00.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary9 up+unknown 2024-09-18T00:21:00.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:21:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:21:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:21:00.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:00.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:21:00.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:21:00.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8d545589-7b5b-44e3-b6e8-4df0695358a6 2024-09-18T00:21:00.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:21:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:21:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:21:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:00.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:00.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:00.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:00.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:00.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:00.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:00.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary7 up+unknown 2024-09-18T00:21:00.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:00.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:00.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:21:00.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:00.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:00.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:00.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:21:00.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:21:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:21:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:21:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:21:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:21:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:00.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:00.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:00.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:21:00.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:00.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:21:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary8 up+unknown 2024-09-18T00:21:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:00.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:21:00.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:00.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:00.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:00.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:21:00.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:21:00.119 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:21:00.115+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d67ae63860 2024-09-18T00:21:00.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:00.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:00.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:21:00.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:00.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:21:00.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:00.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:00.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:00.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary5 up+unknown 2024-09-18T00:21:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:21:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:21:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:21:00.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:21:00.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:21:00.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:00.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:21:00.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:00.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:00.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:00.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:00.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:00.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:21:00.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:21:00.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 684b85b9-e82c-49e4-af6b-eb72455a5948 2024-09-18T00:21:00.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:00.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:00.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:00.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:21:00.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:00.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:21:00.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:00.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:00.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:21:00.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:00.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:00.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:00.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:00.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:00.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:00.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:00.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:21:00.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:21:00.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:00.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:21:00.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:00.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:21:00.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:00.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:00.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:21:00.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:00.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:21:00.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:00.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:00.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:00.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary1 up+unknown 2024-09-18T00:21:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:21:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:00.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:21:00.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:21:00.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary10 up+unknown '' '' 2024-09-18T00:21:00.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:00.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:00.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:21:00.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:00.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:00.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:00.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:00.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:00.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary10 2024-09-18T00:21:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary4 up+unknown '' '' 2024-09-18T00:21:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:21:01.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:01.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:01.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary4 2024-09-18T00:21:01.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary9 up+unknown '' '' 2024-09-18T00:21:01.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:21:01.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:01.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:01.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary9 2024-09-18T00:21:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary7 up+unknown '' '' 2024-09-18T00:21:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:21:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:01.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary7 2024-09-18T00:21:01.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:21:01.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a99a3dae-84ba-4485-a01d-736bffc432b5 2024-09-18T00:21:01.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:01.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:21:01.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:01.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:21:01.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:21:01.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:01.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:01.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:01.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:01.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:01.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:01.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:01.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:01.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:01.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary10 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:21:01.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:21:01.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:21:01.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary8 up+unknown '' '' 2024-09-18T00:21:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:21:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:01.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary8 2024-09-18T00:21:01.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:21:01.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 158b22b4-54b8-411f-9e7e-2163182393ca 2024-09-18T00:21:01.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:21:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:21:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:21:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:01.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:01.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:01.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:01.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:01.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:01.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:01.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:01.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:01.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:01.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary4 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:21:01.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:21:01.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:21:01.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:21:01.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 79b8265e-910e-4b6e-9e5c-df55997a9d65 2024-09-18T00:21:01.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:01.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:21:01.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:01.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:21:01.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:21:01.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:01.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:01.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:01.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:01.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:01.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:01.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:01.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:01.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:01.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:01.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:21:01.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 0c2b99dd-43bc-4302-bae9-69b6a68f20e6 2024-09-18T00:21:01.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:21:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:21:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:21:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary7 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:21:01.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:21:01.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:21:01.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:01.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:01.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:01.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:01.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:01.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:01.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:01.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:01.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:01.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:01.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:01.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary9 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:21:01.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:21:01.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:21:01.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary5 up+unknown '' '' 2024-09-18T00:21:01.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:21:01.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:01.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:01.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary5 2024-09-18T00:21:01.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:01.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:21:01.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 59ac8874-602c-4836-b596-7e605198b3a1 2024-09-18T00:21:01.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:01.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:21:01.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:01.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:21:01.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:21:01.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:01.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:01.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:01.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:01.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:01.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:01.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:01.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:01.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:01.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:01.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary8 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:21:01.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:21:01.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:21:01.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:01.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:01.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:01.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:21:01.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8d545589-7b5b-44e3-b6e8-4df0695358a6 2024-09-18T00:21:01.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:01.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:21:01.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:01.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:21:01.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:21:01.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:01.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:21:01.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:21:01.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:01.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:01.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary10 2024-09-18T00:21:01.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:21:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:21:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary10 2024-09-18T00:21:01.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:01.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:01.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:01.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:01.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:01.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:01.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:01.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:01.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary5 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:21:01.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:21:01.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:21:01.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:01.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:01.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:01.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:21:01.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:01.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:01.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:21:01.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:21:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary4 2024-09-18T00:21:01.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:21:01.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:21:01.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary4 2024-09-18T00:21:01.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:21:01.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:01.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:01.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:01.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:01.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary7 2024-09-18T00:21:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:21:01.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:21:01.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary7 2024-09-18T00:21:01.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:21:01.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:21:01.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:01.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:01.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary9 2024-09-18T00:21:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:21:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:21:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary9 2024-09-18T00:21:01.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:01.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:01.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:21:01.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:21:01.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:01.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:01.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary8 2024-09-18T00:21:01.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:21:01.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:21:01.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary8 2024-09-18T00:21:01.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:01.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:01.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:21:01.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:21:01.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:01.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:01.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary5 2024-09-18T00:21:01.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:21:01.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:21:01.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary5 2024-09-18T00:21:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary1 up+unknown '' '' 2024-09-18T00:21:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:21:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary1 2024-09-18T00:21:01.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:21:01.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 684b85b9-e82c-49e4-af6b-eb72455a5948 2024-09-18T00:21:01.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:01.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:21:01.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:01.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:21:01.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:01.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:01.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:21:01.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:01.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:01.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:01.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:01.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:01.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:01.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:01.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:01.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:01.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:01.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary1 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:21:01.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:21:01.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:01.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:21:01.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:01.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:01.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:01.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:20:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:01.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:01.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:21:01.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:01.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:21:01.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:01.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:01.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:01.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:01.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary1 2024-09-18T00:21:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:21:01.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:21:02.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary1 2024-09-18T00:21:02.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:21:02.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:21:02.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:21:02.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:21:02.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:21:02.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:21:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:21:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:21:02.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:21:02.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:21:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:21:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:21:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:21:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:21:02.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:21:02.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:21:02.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:21:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:21:02.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd1 2024-09-18T00:21:02.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:21:02.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:21:02.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd2 2024-09-18T00:21:02.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:21:02.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:21:02.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd3 2024-09-18T00:21:02.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:21:02.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:21:02.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd4 2024-09-18T00:21:02.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:21:02.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:21:02.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd5 2024-09-18T00:21:02.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:21:02.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:21:02.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd6 2024-09-18T00:21:02.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:21:02.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:21:02.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T00:21:02.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:21:02.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:02.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:21:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:21:02.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:21:02.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:21:02.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:02.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:21:02.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:02.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:02.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:02.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:02.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:02.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:21:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:21:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 74d7ffd9-fbb0-4520-9d48-784c6cd85819 2024-09-18T00:21:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:21:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:21:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:21:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:02.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:02.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:02.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:02.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:02.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:02.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:02.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:21:02.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4 2024-09-18T00:21:02.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:02.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:02.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:02.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59 2024-09-18T00:21:02.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:02.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:21:02.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:02.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:02.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:20:59' 2024-09-18T00:21:02.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:02.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:02.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:02.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:02.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:21:02.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:21:02.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:02.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:21:02.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:02.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:21:02.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:02.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:02.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:02.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:02.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:02.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:02.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:02.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:02.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:02.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:02.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:02.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:02.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+unknown '2024-09-18 00:20:59' 2024-09-18T00:21:02.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:21:02.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:02.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:21:02.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:02.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:20:59' 2024-09-18T00:21:02.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:02.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:02.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:21:03.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:21:03.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:21:03.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:03.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:03.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:21:03.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:03.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:21:03.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:03.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:03.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:03.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:03.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:03.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary3 up+unknown 2024-09-18T00:21:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:21:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:21:03.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:21:03.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:20:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:20:59' 2024-09-18T00:21:03.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:03.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:21:03.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:20:59' 2024-09-18T00:21:03.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:21:03.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:03.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:03.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:03.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:03.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:03.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:03.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary2 up+unknown 2024-09-18T00:21:03.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:03.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:03.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:21:03.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:03.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:03.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:03.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:21:03.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:21:03.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd7 2024-09-18T00:21:03.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:21:03.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:21:04.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary3 up+unknown '' '' 2024-09-18T00:21:04.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:04.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:04.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:21:04.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:04.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:04.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:04.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:04.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:04.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary3 2024-09-18T00:21:04.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary2 up+unknown '' '' 2024-09-18T00:21:04.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:04.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:04.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:21:04.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:04.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:21:04.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:21:04.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:04.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:04.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary2 2024-09-18T00:21:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:21:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 74d7ffd9-fbb0-4520-9d48-784c6cd85819 2024-09-18T00:21:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:04.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:04.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:04.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:21:04 2024-09-18T00:21:04.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:04.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:21:04.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:04.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:04.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:21:04' 2024-09-18T00:21:04.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:04.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:04.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:04.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:04.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:04.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:04.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:04.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:04.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:04.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:04.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:21:04' 2024-09-18T00:21:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary3 up+unknown '2024-09-18 00:21:04' 2024-09-18T00:21:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:21:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:21:04' 2024-09-18T00:21:04.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:04.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:04.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:04.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:21:04.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4 2024-09-18T00:21:04.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:04.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:04.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:21:04.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:21:04 2024-09-18T00:21:04.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:21:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:21:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:21:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:21:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:21:04' 2024-09-18T00:21:04.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:21:04.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:21:04.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:21:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:21:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:21:04.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:04.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:21:04.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:04.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:21:04.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:21:04.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:21:04.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:21:04.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:21:04' 2024-09-18T00:21:04.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary2 up+unknown '2024-09-18 00:21:04' 2024-09-18T00:21:04.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:04.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:04.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:21:04.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:21:04.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:21:04' 2024-09-18T00:21:04.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:21:04.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:21:04.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:21:04.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING2842614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618862,"replay_state":"idle"}2024-09-18 00:21:04image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}2024-09-18 00:21:04image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:21:04image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:21:04image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}2024-09-18 00:21:04image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}2024-09-18 00:21:04image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}2024-09-18 00:21:04image-primary859ac8874-602c-4836-b596-7e605198b3a1up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}2024-09-18 00:21:04image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}2024-09-18 00:21:04' 2024-09-18T00:21:04.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:04.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:21:04.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:21:04' 2024-09-18T00:21:04.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:21:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:04.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:04.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:04.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary3 2024-09-18T00:21:04.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:04.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:04.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:21:04.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:21:04.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary3 2024-09-18T00:21:05.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING2842614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618862,"replay_state":"idle"}2024-09-18 00:21:04image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}2024-09-18 00:21:04image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:21:04image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:21:04image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}2024-09-18 00:21:04image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}2024-09-18 00:21:04image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}2024-09-18 00:20:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}2024-09-18 00:21:04image-primary859ac8874-602c-4836-b596-7e605198b3a1up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}2024-09-18 00:21:04image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:21:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}2024-09-18 00:21:04' 2024-09-18T00:21:05.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:21:05.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:21:05.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:21:04' 2024-09-18T00:21:05.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:21:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:21:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:21:05.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:21:05.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:21:05.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:05.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:21:05.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary2 2024-09-18T00:21:05.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:21:05.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:21:05.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:21:05.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:21:05.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary2 2024-09-18T00:21:06.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:21:06.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:21:06.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:21:06.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:21:06.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:21:06.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:21:06.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd8 2024-09-18T00:21:06.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:21:06.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:21:06.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd9 2024-09-18T00:21:06.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:21:06.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:21:19.226 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:21:19.227+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735aedd0 2024-09-18T00:21:19.254 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:21:19.255+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735aedd0 2024-09-18T00:22:02.003 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:22:01.998+0000 7fd4c158b640 -1 reset not still connected to 0x56436346f6c0 2024-09-18T00:22:43.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=d58f036957c1e0f3abdd0c642f8a89bd 2024-09-18T00:22:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd0 2024-09-18T00:22:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ d58f036957c1e0f3abdd0c642f8a89bd != \d\5\8\f\0\3\6\9\5\7\c\1\e\0\f\3\a\b\d\d\0\c\6\4\2\f\8\a\8\9\b\d ]] 2024-09-18T00:22:51.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=9a73956cf5e56c57066849d632e1cad7 2024-09-18T00:22:51.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd5 2024-09-18T00:22:52.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=dc9d4c9ae53d7213d676da0b95a097a3 2024-09-18T00:22:52.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd3 2024-09-18T00:22:52.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=f3b60fabbd942f607ea139df31608403 2024-09-18T00:22:52.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd1 2024-09-18T00:22:52.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 9a73956cf5e56c57066849d632e1cad7 != \9\a\7\3\9\5\6\c\f\5\e\5\6\c\5\7\0\6\6\8\4\9\d\6\3\2\e\1\c\a\d\7 ]] 2024-09-18T00:22:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=bf248b7b84196905c9cb30e67a56b6ee 2024-09-18T00:22:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd4 2024-09-18T00:22:52.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=ca5c43a28591836e4aa15c06986aee5a 2024-09-18T00:22:52.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd2 2024-09-18T00:22:53.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ f3b60fabbd942f607ea139df31608403 != \f\3\b\6\0\f\a\b\b\d\9\4\2\f\6\0\7\e\a\1\3\9\d\f\3\1\6\0\8\4\0\3 ]] 2024-09-18T00:22:53.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ dc9d4c9ae53d7213d676da0b95a097a3 != \d\c\9\d\4\c\9\a\e\5\3\d\7\2\1\3\d\6\7\6\d\a\0\b\9\5\a\0\9\7\a\3 ]] 2024-09-18T00:22:53.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=7ebd324896096b75aaa5dacbc10e0196 2024-09-18T00:22:53.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd7 2024-09-18T00:22:53.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=966300c16ad66f152c865ef9433a8c78 2024-09-18T00:22:53.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd6 2024-09-18T00:22:53.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ bf248b7b84196905c9cb30e67a56b6ee != \b\f\2\4\8\b\7\b\8\4\1\9\6\9\0\5\c\9\c\b\3\0\e\6\7\a\5\6\b\6\e\e ]] 2024-09-18T00:22:53.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ ca5c43a28591836e4aa15c06986aee5a != \c\a\5\c\4\3\a\2\8\5\9\1\8\3\6\e\4\a\a\1\5\c\0\6\9\8\6\a\e\e\5\a ]] 2024-09-18T00:22:54.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 7ebd324896096b75aaa5dacbc10e0196 != \7\e\b\d\3\2\4\8\9\6\0\9\6\b\7\5\a\a\a\5\d\a\c\b\c\1\0\e\0\1\9\6 ]] 2024-09-18T00:22:54.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:22:54.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 71141 2024-09-18T00:22:54.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 966300c16ad66f152c865ef9433a8c78 != \9\6\6\3\0\0\c\1\6\a\d\6\6\f\1\5\2\c\8\6\5\e\f\9\4\3\3\a\8\c\7\8 ]] 2024-09-18T00:22:54.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=c9a7d8902d05cc5151a7e74576ef32de 2024-09-18T00:22:54.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd8 2024-09-18T00:22:55.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=65dcc38740816803f8112679a1bdff56 2024-09-18T00:22:55.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd9 2024-09-18T00:22:55.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ c9a7d8902d05cc5151a7e74576ef32de != \c\9\a\7\d\8\9\0\2\d\0\5\c\c\5\1\5\1\a\7\e\7\4\5\7\6\e\f\3\2\d\e ]] 2024-09-18T00:22:55.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 65dcc38740816803f8112679a1bdff56 != \6\5\d\c\c\3\8\7\4\0\8\1\6\8\0\3\f\8\1\1\2\6\7\9\a\1\b\d\f\f\5\6 ]] 2024-09-18T00:22:55.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:22:55.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 71142 2024-09-18T00:22:55.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:22:55.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 71144 2024-09-18T00:22:55.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:22:55.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 71147 2024-09-18T00:22:55.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:22:55.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 71149 2024-09-18T00:22:55.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:22:55.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 71151 2024-09-18T00:22:55.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:22:55.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 71153 2024-09-18T00:22:55.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:22:55.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 71155 2024-09-18T00:22:55.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:22:55.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 71156 2024-09-18T00:22:55.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:22:55.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:22:55.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:22:55.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary1 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+replaying 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:22:55.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:22:56.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+replaying '' '' 2024-09-18T00:22:56.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:22:56.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:22:56.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:22:56.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:22:56.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:22:56.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:22:56.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:22:56.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:22:56.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:22:56.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:22:56.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 684b85b9-e82c-49e4-af6b-eb72455a5948 2024-09-18T00:22:56.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:22:56.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72794112.0,"last_snapshot_bytes":72794112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618862,"remote_snapshot_timestamp":1726618862,"replay_state":"idle"} 2024-09-18T00:22:56.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:22:56.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:22:39 2024-09-18T00:22:56.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:22:56.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:22:56.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:22:56.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:22:56.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:22:54' 2024-09-18T00:22:56.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:22:56.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:22:56.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:22:56.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72794112.0,"last_snapshot_bytes":72794112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618862,"remote_snapshot_timestamp":1726618862,"replay_state":"idle"} 2024-09-18T00:22:56.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:22:56.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:22:56.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:22:56.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:22:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:22:56.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:22:56.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:22:56.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:22:56.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:22:39' 2024-09-18T00:22:56.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+replaying '2024-09-18 00:22:39' 2024-09-18T00:22:56.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:22:56.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:22:56.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:22:56.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:22:56.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:22:39' 2024-09-18T00:22:56.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:22:56.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:22:56.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:22:57.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1684b85b9-e82c-49e4-af6b-eb72455a5948up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72794112.0,"last_snapshot_bytes":72794112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618862,"remote_snapshot_timestamp":1726618862,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:54image-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70070272.0,"last_snapshot_bytes":70070272,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:54image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618866,"remote_snapshot_timestamp":1726618866,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:54image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":95211520.0,"last_snapshot_bytes":95211520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618866,"remote_snapshot_timestamp":1726618866,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:54image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60440576.0,"last_snapshot_bytes":60440576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:54image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70135808.0,"last_snapshot_bytes":70135808,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:54image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70066176.0,"last_snapshot_bytes":70066176,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618858,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:54image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62324736.0,"last_snapshot_bytes":62324736,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:54image-primary859ac8874-602c-4836-b596-7e605198b3a1up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72065024.0,"last_snapshot_bytes":72065024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:54image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70205440.0,"last_snapshot_bytes":70205440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:54' 2024-09-18T00:22:57.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:22:57.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:22:57.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:22:39' 2024-09-18T00:22:57.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:22:57.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:22:57.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:22:57.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:22:57.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:22:57.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:22:57.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:22:57.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary1 2024-09-18T00:22:57.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:22:57.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:22:57.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:22:57.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary1 2024-09-18T00:22:57.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary1 2024-09-18T00:22:59.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:22:59.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary1 2024-09-18T00:22:59.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:22:59.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:22:59.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:22:59.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:22:59.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:22:59.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary1 2024-09-18T00:22:59.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:22:59.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary1 2024-09-18T00:22:59.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:22:59.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:22:59.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary2 2024-09-18T00:22:59.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:22:59.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:22:59.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:22:59.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:22:59.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+replaying 2024-09-18T00:22:59.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:22:59.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:22:59.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:22:59.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:22:59.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:22:59.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:22:59.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:22:59.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:23:00.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+replaying '' '' 2024-09-18T00:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:00.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:00.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:00.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:23:00.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:23:00.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4 2024-09-18T00:23:00.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:23:00.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618866,"remote_snapshot_timestamp":1726618866,"replay_state":"idle"} 2024-09-18T00:23:00.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:23:00.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:22:39 2024-09-18T00:23:00.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:23:00.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:23:00.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:23:00.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:23:00.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:22:59' 2024-09-18T00:23:00.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:23:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:23:00.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:23:00.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618866,"remote_snapshot_timestamp":1726618866,"replay_state":"idle"} 2024-09-18T00:23:00.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:23:00.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:00.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:23:00.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:00.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:23:00.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:23:00.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:23:00.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:23:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:22:39' 2024-09-18T00:23:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+replaying '2024-09-18 00:22:39' 2024-09-18T00:23:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:22:39' 2024-09-18T00:23:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:00.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:00.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:23:01.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70070272.0,"last_snapshot_bytes":70070272,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:59image-primary2cb5b4d45-db5c-42d5-ae01-495a7f5d0ff4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618866,"remote_snapshot_timestamp":1726618866,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:59image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":95211520.0,"last_snapshot_bytes":95211520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618866,"remote_snapshot_timestamp":1726618866,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:59image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60440576.0,"last_snapshot_bytes":60440576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:59image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70135808.0,"last_snapshot_bytes":70135808,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:59image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70066176.0,"last_snapshot_bytes":70066176,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618858,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:59image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62324736.0,"last_snapshot_bytes":62324736,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:59image-primary859ac8874-602c-4836-b596-7e605198b3a1up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72065024.0,"last_snapshot_bytes":72065024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:59image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70205440.0,"last_snapshot_bytes":70205440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:22:59' 2024-09-18T00:23:01.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:23:01.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:23:01.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:22:39' 2024-09-18T00:23:01.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:23:01.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:23:01.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:01.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:23:01.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:01.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:01.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:01.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary2 2024-09-18T00:23:01.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:23:01.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:01.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:01.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary2 2024-09-18T00:23:01.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary2 2024-09-18T00:23:03.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:23:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary2 2024-09-18T00:23:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:23:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary2 2024-09-18T00:23:03.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:03.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary2 2024-09-18T00:23:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:23:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary3 2024-09-18T00:23:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:23:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+replaying 2024-09-18T00:23:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:03.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:23:04.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+replaying '' '' 2024-09-18T00:23:04.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:04.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:04.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:04.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:04.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:04.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:04.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:04.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:04.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:23:04.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:23:04.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 74d7ffd9-fbb0-4520-9d48-784c6cd85819 2024-09-18T00:23:04.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:23:04.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":95211520.0,"last_snapshot_bytes":95211520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618866,"remote_snapshot_timestamp":1726618866,"replay_state":"idle"} 2024-09-18T00:23:04.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:23:04.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:22:39 2024-09-18T00:23:04.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:23:04.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:23:04.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:23:04.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:23:04.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:04' 2024-09-18T00:23:04.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:23:04.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:23:04.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:23:04.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":95211520.0,"last_snapshot_bytes":95211520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618866,"remote_snapshot_timestamp":1726618866,"replay_state":"idle"} 2024-09-18T00:23:04.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:23:04.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:04.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:23:04.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:04.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:23:04.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:23:04.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:23:04.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:23:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:22:39' 2024-09-18T00:23:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+replaying '2024-09-18 00:22:39' 2024-09-18T00:23:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:22:39' 2024-09-18T00:23:04.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:04.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:04.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:23:04.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK842544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70070272.0,"last_snapshot_bytes":70070272,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary374d7ffd9-fbb0-4520-9d48-784c6cd85819up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":95211520.0,"last_snapshot_bytes":95211520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618866,"remote_snapshot_timestamp":1726618866,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60440576.0,"last_snapshot_bytes":60440576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70135808.0,"last_snapshot_bytes":70135808,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70066176.0,"last_snapshot_bytes":70066176,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618858,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62324736.0,"last_snapshot_bytes":62324736,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary859ac8874-602c-4836-b596-7e605198b3a1up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72065024.0,"last_snapshot_bytes":72065024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70205440.0,"last_snapshot_bytes":70205440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04' 2024-09-18T00:23:04.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:23:04.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:23:04.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:22:39' 2024-09-18T00:23:04.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:23:04.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:23:04.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:04.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:23:04.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:04.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:04.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:04.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary3 2024-09-18T00:23:04.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:23:04.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:04.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:04.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary3 2024-09-18T00:23:05.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary3 2024-09-18T00:23:07.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:23:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary3 2024-09-18T00:23:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:23:07.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary3 2024-09-18T00:23:07.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:07.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary3 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary4 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+replaying 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:07.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:07.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:07.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:07.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:23:08.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+replaying '' '' 2024-09-18T00:23:08.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:08.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:08.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:08.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:08.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:08.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:08.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:08.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:08.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:23:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:23:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 158b22b4-54b8-411f-9e7e-2163182393ca 2024-09-18T00:23:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:23:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60440576.0,"last_snapshot_bytes":60440576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"} 2024-09-18T00:23:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:23:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:22:39 2024-09-18T00:23:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:23:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:23:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:23:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:23:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:04' 2024-09-18T00:23:08.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:23:08.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:23:08.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:23:08.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60440576.0,"last_snapshot_bytes":60440576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"} 2024-09-18T00:23:08.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:23:08.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:08.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:23:08.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:08.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:23:08.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:23:08.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:23:08.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:23:08.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:22:39' 2024-09-18T00:23:08.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+replaying '2024-09-18 00:22:39' 2024-09-18T00:23:08.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:08.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:08.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:08.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:08.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:22:39' 2024-09-18T00:23:08.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:08.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:08.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:23:08.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70070272.0,"last_snapshot_bytes":70070272,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary4158b22b4-54b8-411f-9e7e-2163182393caup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60440576.0,"last_snapshot_bytes":60440576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70135808.0,"last_snapshot_bytes":70135808,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70066176.0,"last_snapshot_bytes":70066176,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618858,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62324736.0,"last_snapshot_bytes":62324736,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary859ac8874-602c-4836-b596-7e605198b3a1up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72065024.0,"last_snapshot_bytes":72065024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70205440.0,"last_snapshot_bytes":70205440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:22:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:04' 2024-09-18T00:23:08.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:23:08.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:23:08.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:22:39' 2024-09-18T00:23:08.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:23:08.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:23:08.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:08.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:23:08.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:08.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:08.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:08.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary4 2024-09-18T00:23:08.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:23:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary4 2024-09-18T00:23:08.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary4 2024-09-18T00:23:10.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:23:10.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary4 2024-09-18T00:23:10.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:10.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:10.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:10.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:10.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:23:10.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary4 2024-09-18T00:23:10.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:10.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary4 2024-09-18T00:23:10.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:10.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:23:10.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary5 2024-09-18T00:23:10.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:10.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:10.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:10.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:23:10.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+replaying 2024-09-18T00:23:10.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:10.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:10.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:10.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:10.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:10.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:10.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:10.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:23:11.210 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:23:11.203+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d685b86750 2024-09-18T00:23:11.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+replaying '' '' 2024-09-18T00:23:11.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:11.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:11.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:11.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:11.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:11.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:11.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:11.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:11.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:23:11.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:23:11.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8d545589-7b5b-44e3-b6e8-4df0695358a6 2024-09-18T00:23:11.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:23:11.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70135808.0,"last_snapshot_bytes":70135808,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"} 2024-09-18T00:23:11.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:23:11.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:09 2024-09-18T00:23:11.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:23:11.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:23:11.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:23:11.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:23:11.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:09' 2024-09-18T00:23:11.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:23:11.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:23:11.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:23:11.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70135808.0,"last_snapshot_bytes":70135808,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"} 2024-09-18T00:23:11.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:23:11.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:11.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:23:11.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:11.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:23:11.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:23:11.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:23:11.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:23:11.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:23:09' 2024-09-18T00:23:11.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+replaying '2024-09-18 00:23:09' 2024-09-18T00:23:11.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:11.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:11.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:11.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:11.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:23:09' 2024-09-18T00:23:11.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:11.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:11.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:23:11.470 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:23:11.471+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d68015a1a0 2024-09-18T00:23:11.471 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:23:11.471+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d685b86750 2024-09-18T00:23:11.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK642544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70070272.0,"last_snapshot_bytes":70070272,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:09image-primary58d545589-7b5b-44e3-b6e8-4df0695358a6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70135808.0,"last_snapshot_bytes":70135808,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:09image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70066176.0,"last_snapshot_bytes":70066176,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618858,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:09image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62324736.0,"last_snapshot_bytes":62324736,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:09image-primary859ac8874-602c-4836-b596-7e605198b3a1up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72065024.0,"last_snapshot_bytes":72065024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:09image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70205440.0,"last_snapshot_bytes":70205440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:09' 2024-09-18T00:23:11.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:23:11.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:23:11.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:23:09' 2024-09-18T00:23:11.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:23:11.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:23:11.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:11.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:23:11.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary5 2024-09-18T00:23:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:23:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:11.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary5 2024-09-18T00:23:11.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary5 2024-09-18T00:23:13.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:23:13.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary5 2024-09-18T00:23:13.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:13.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:13.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:23:13.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary5 2024-09-18T00:23:13.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:13.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary5 2024-09-18T00:23:13.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary6 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+replaying 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:13.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:13.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:13.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:23:14.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+replaying '' '' 2024-09-18T00:23:14.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:14.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:14.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:14.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:14.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:14.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:14.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:14.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:14.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:23:14.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:23:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 88c5b661-6663-4bbd-986c-9e5a7fb24308 2024-09-18T00:23:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:23:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70066176.0,"last_snapshot_bytes":70066176,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618858,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"} 2024-09-18T00:23:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:23:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:09 2024-09-18T00:23:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:23:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:23:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:23:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:23:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:14' 2024-09-18T00:23:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:23:14.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:23:14.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:23:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70066176.0,"last_snapshot_bytes":70066176,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618858,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"} 2024-09-18T00:23:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:23:14.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:14.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:23:14.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:14.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:23:14.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:23:14.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:23:14.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:23:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:23:09' 2024-09-18T00:23:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+replaying '2024-09-18 00:23:09' 2024-09-18T00:23:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:23:09' 2024-09-18T00:23:14.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:14.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:14.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:23:15.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70070272.0,"last_snapshot_bytes":70070272,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:14image-primary688c5b661-6663-4bbd-986c-9e5a7fb24308up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70066176.0,"last_snapshot_bytes":70066176,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618858,"remote_snapshot_timestamp":1726618858,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:14image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62324736.0,"last_snapshot_bytes":62324736,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:14image-primary859ac8874-602c-4836-b596-7e605198b3a1up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72065024.0,"last_snapshot_bytes":72065024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:14image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70205440.0,"last_snapshot_bytes":70205440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:14' 2024-09-18T00:23:15.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:23:15.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:23:15.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:23:09' 2024-09-18T00:23:15.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:23:15.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:23:15.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:15.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:23:15.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary6 2024-09-18T00:23:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:23:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:15.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary6 2024-09-18T00:23:15.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary6 2024-09-18T00:23:17.230 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:23:17.227+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:23:17.230 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:23:17.231+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:23:17.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:23:17.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary6 2024-09-18T00:23:17.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:17.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:17.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:17.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:17.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:23:17.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary6 2024-09-18T00:23:17.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:17.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary6 2024-09-18T00:23:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:23:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary7 2024-09-18T00:23:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:23:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+replaying 2024-09-18T00:23:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:17.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:17.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:17.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:17.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:23:18.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+replaying '' '' 2024-09-18T00:23:18.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:18.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:18.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:18.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:18.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:18.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:18.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:18.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:18.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:23:18.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:23:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 79b8265e-910e-4b6e-9e5c-df55997a9d65 2024-09-18T00:23:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:23:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62324736.0,"last_snapshot_bytes":62324736,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"} 2024-09-18T00:23:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:23:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:09 2024-09-18T00:23:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:23:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:23:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:23:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:23:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:14' 2024-09-18T00:23:18.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:23:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:23:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:23:18.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62324736.0,"last_snapshot_bytes":62324736,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"} 2024-09-18T00:23:18.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:23:18.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:23:18.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:18.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:23:18.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:23:18.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:23:18.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:23:18.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:23:09' 2024-09-18T00:23:18.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+replaying '2024-09-18 00:23:09' 2024-09-18T00:23:18.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:23:09' 2024-09-18T00:23:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:23:18.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70070272.0,"last_snapshot_bytes":70070272,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:14image-primary779b8265e-910e-4b6e-9e5c-df55997a9d65up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62324736.0,"last_snapshot_bytes":62324736,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:14image-primary859ac8874-602c-4836-b596-7e605198b3a1up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72065024.0,"last_snapshot_bytes":72065024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:14image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70205440.0,"last_snapshot_bytes":70205440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:14' 2024-09-18T00:23:18.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:23:18.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:23:18.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:23:09' 2024-09-18T00:23:18.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:23:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:23:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:23:18.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary7 2024-09-18T00:23:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:23:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:18.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary7 2024-09-18T00:23:18.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary7 2024-09-18T00:23:20.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:23:20.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary7 2024-09-18T00:23:20.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:20.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:20.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:20.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:20.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:23:20.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary7 2024-09-18T00:23:20.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:20.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary7 2024-09-18T00:23:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:23:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary8 2024-09-18T00:23:20.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:23:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+replaying 2024-09-18T00:23:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:20.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:20.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:20.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:20.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:20.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:20.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:23:20.632 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:23:20.635+0000 7fac32fac640 -1 reset not still connected to 0x55a83a5192b0 2024-09-18T00:23:20.635 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:23:20.635+0000 7fac32fac640 -1 reset not still connected to 0x55a83a519e10 2024-09-18T00:23:21.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+replaying '' '' 2024-09-18T00:23:21.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:21.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:21.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:21.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:21.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:21.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:21.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:21.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:21.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:23:21.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:23:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 59ac8874-602c-4836-b596-7e605198b3a1 2024-09-18T00:23:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:23:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72065024.0,"last_snapshot_bytes":72065024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"} 2024-09-18T00:23:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:23:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:09 2024-09-18T00:23:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:23:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:23:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:23:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:23:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:19' 2024-09-18T00:23:21.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:23:21.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:23:21.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:23:21.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72065024.0,"last_snapshot_bytes":72065024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"} 2024-09-18T00:23:21.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:23:21.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:21.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:23:21.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:21.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:23:21.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:23:21.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:23:21.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:23:21.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:23:09' 2024-09-18T00:23:21.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+replaying '2024-09-18 00:23:09' 2024-09-18T00:23:21.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:21.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:21.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:21.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:21.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:23:09' 2024-09-18T00:23:21.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:21.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:21.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:23:21.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK342544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70070272.0,"last_snapshot_bytes":70070272,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:19image-primary859ac8874-602c-4836-b596-7e605198b3a1up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72065024.0,"last_snapshot_bytes":72065024,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:19image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70205440.0,"last_snapshot_bytes":70205440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:19' 2024-09-18T00:23:21.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:23:21.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:23:21.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:23:09' 2024-09-18T00:23:21.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:23:21.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:23:21.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:21.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:23:21.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:21.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:21.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:21.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary8 2024-09-18T00:23:21.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:23:21.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:21.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:21.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary8 2024-09-18T00:23:21.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary8 2024-09-18T00:23:23.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:23:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary8 2024-09-18T00:23:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:23.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:23:23.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary8 2024-09-18T00:23:23.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:23.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary8 2024-09-18T00:23:23.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary9 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+replaying 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:23.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:23:24.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+replaying '' '' 2024-09-18T00:23:24.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:24.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:24.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:24.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:24.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:24.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:24.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:24.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:24.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:23:24.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:23:24.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 0c2b99dd-43bc-4302-bae9-69b6a68f20e6 2024-09-18T00:23:24.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:23:24.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70205440.0,"last_snapshot_bytes":70205440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"} 2024-09-18T00:23:24.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:23:24.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:09 2024-09-18T00:23:24.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:23:24.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:23:24.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:23:24.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:23:24.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:24' 2024-09-18T00:23:24.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:23:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:23:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:23:24.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70205440.0,"last_snapshot_bytes":70205440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"} 2024-09-18T00:23:24.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:23:24.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:24.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:23:24.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:24.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:23:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:23:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:23:24.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:23:24.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:23:09' 2024-09-18T00:23:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+replaying '2024-09-18 00:23:09' 2024-09-18T00:23:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:23:09' 2024-09-18T00:23:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:24.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:23:24.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70070272.0,"last_snapshot_bytes":70070272,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:24image-primary90c2b99dd-43bc-4302-bae9-69b6a68f20e6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70205440.0,"last_snapshot_bytes":70205440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:24' 2024-09-18T00:23:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:23:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:23:24.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:23:09' 2024-09-18T00:23:24.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:23:24.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:23:24.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:24.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:23:24.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:24.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:24.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:24.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary9 2024-09-18T00:23:24.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:23:24.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:24.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:24.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary9 2024-09-18T00:23:24.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary9 2024-09-18T00:23:26.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:23:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary9 2024-09-18T00:23:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:26.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:26.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:23:26.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary9 2024-09-18T00:23:26.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:26.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary9 2024-09-18T00:23:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:23:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary10 2024-09-18T00:23:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:23:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+replaying 2024-09-18T00:23:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:26.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:23:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+replaying '' '' 2024-09-18T00:23:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:23:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:23:27.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:27.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:27.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a99a3dae-84ba-4485-a01d-736bffc432b5 2024-09-18T00:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70070272.0,"last_snapshot_bytes":70070272,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"} 2024-09-18T00:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:09 2024-09-18T00:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:23:24' 2024-09-18T00:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:23:27.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:23:27.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70070272.0,"last_snapshot_bytes":70070272,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"} 2024-09-18T00:23:27.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:23:27.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:27.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:23:27.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:27.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:23:27.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:23:27.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:23:27.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:23:27.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:23:09' 2024-09-18T00:23:27.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+replaying '2024-09-18 00:23:09' 2024-09-18T00:23:27.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:27.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:27.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:27.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:23:27.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:23:09' 2024-09-18T00:23:27.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:23:27.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:23:27.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:23:27.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10a99a3dae-84ba-4485-a01d-736bffc432b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70070272.0,"last_snapshot_bytes":70070272,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726618861,"remote_snapshot_timestamp":1726618861,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:23:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:23:24' 2024-09-18T00:23:27.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:23:27.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:23:27.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:23:09' 2024-09-18T00:23:27.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:23:27.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:23:27.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:23:27.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:23:27.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:23:27.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:27.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:27.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary10 2024-09-18T00:23:27.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:23:27.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:27.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:27.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary10 2024-09-18T00:23:27.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary10 2024-09-18T00:23:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:23:29.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary10 2024-09-18T00:23:29.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:29.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:29.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:29.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:23:29.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:23:29.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary10 2024-09-18T00:23:29.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:29.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary10 2024-09-18T00:23:29.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..10} 2024-09-18T00:23:29.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS=() 2024-09-18T00:23:29.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS=() 2024-09-18T00:23:29.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS=() 2024-09-18T00:23:29.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS=() 2024-09-18T00:23:29.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RET=0 2024-09-18T00:23:29.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary1 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary1 snapshot 10G 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:29.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary1 10G 2024-09-18T00:23:29.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:29.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:29.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:29.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:29.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:23:29.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:29.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:23:29.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:23:29.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:23:29.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary1 2024-09-18T00:23:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:23:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:23:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary1 snapshot 2024-09-18T00:23:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:23:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary1 snapshot 2024-09-18T00:23:30.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:23:30.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary1 2024-09-18T00:23:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary1': 2024-09-18T00:23:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:23:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:23:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:23:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 22bd65201bf3 2024-09-18T00:23:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.22bd65201bf3 2024-09-18T00:23:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:23:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:23:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:23:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:23:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:23:29 2024 2024-09-18T00:23:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:23:29 2024 2024-09-18T00:23:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:23:29 2024 2024-09-18T00:23:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:23:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:23:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: d18e1596-14e2-4da8-9e9a-daeec5ecfa30 2024-09-18T00:23:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:23:30.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:23:30.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:23:30.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd0 2024-09-18T00:23:30.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:23:30.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd0 2024-09-18T00:23:30.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:23:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:23:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:23:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 3365e9dc-3155-4653-882c-e82dfff54169 2024-09-18T00:23:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:23:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:23:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:23:30.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:23:31.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:23:31.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:23:31.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:31.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary1 2024-09-18T00:23:31.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd0 test-primary1 2024-09-18T00:23:31.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:23:31.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary1 2024-09-18T00:23:31.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:23:31.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary1 2024-09-18T00:23:31.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:31.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:31.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:23:31.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:31.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:23:31.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:31.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:31.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:23:31.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:23:31.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:23:31.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary1 2024-09-18T00:23:31.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:31.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary1 2024-09-18T00:23:31.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary1 2024-09-18T00:23:31.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:23:31.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary2 2024-09-18T00:23:31.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary2 snapshot 10G 2024-09-18T00:23:31.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:31.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:31.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:31.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:31.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:31.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:31.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:31.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:23:31.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:31.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary2 10G 2024-09-18T00:23:31.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:31.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:31.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:31.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:31.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:31.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:31.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:23:31.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:23:31.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:23:31.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:31.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary2 2024-09-18T00:23:31.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:23:31.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:31.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:23:31.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:31.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary2 snapshot 2024-09-18T00:23:31.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:31.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:31.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:31.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:31.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary2 snapshot 2024-09-18T00:23:32.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1325 2024-09-18T00:23:32.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:32.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:23:32.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:23:32.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary1/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary1' 2024-09-18T00:23:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary2 2024-09-18T00:23:32.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary2': 2024-09-18T00:23:32.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:23:32.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:23:32.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:23:32.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 22cb4a20bdae 2024-09-18T00:23:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.22cb4a20bdae 2024-09-18T00:23:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:23:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:23:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:23:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:23:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:23:31 2024 2024-09-18T00:23:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:23:31 2024 2024-09-18T00:23:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:23:31 2024 2024-09-18T00:23:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:23:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:23:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 838864e7-c9c1-4cb7-9af3-e63a599ff8a4 2024-09-18T00:23:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:23:32.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:23:32.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:23:32.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd1 2024-09-18T00:23:32.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:23:32.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd1 2024-09-18T00:23:32.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:23:32.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:23:32.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:23:32.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 4edb2cc2-06a8-4b48-b209-e81626e5d06f 2024-09-18T00:23:32.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:23:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:23:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:23:32.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:23:33.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:23:33.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:23:33.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:33.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary2 2024-09-18T00:23:33.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd1 test-primary2 2024-09-18T00:23:33.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:23:33.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary2 2024-09-18T00:23:33.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:23:33.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary2 2024-09-18T00:23:33.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:23:33.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:33.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:33.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:33.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:33.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:23:33.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary3 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary3 snapshot 10G 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:33.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary3 10G 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary2 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:23:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:23:33.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:33.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary3 2024-09-18T00:23:33.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary2 2024-09-18T00:23:33.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary2 2024-09-18T00:23:33.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:23:33.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:23:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary3 snapshot 2024-09-18T00:23:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary3 snapshot 2024-09-18T00:23:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:23:33.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary2/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary2' 2024-09-18T00:23:34.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1327 2024-09-18T00:23:34.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:23:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:34.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary3 2024-09-18T00:23:34.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary3': 2024-09-18T00:23:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:23:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:23:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:23:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 22dac3185b25 2024-09-18T00:23:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.22dac3185b25 2024-09-18T00:23:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:23:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:23:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:23:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:23:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:23:33 2024 2024-09-18T00:23:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:23:33 2024 2024-09-18T00:23:34.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:23:33 2024 2024-09-18T00:23:34.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:23:34.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:23:34.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: af284746-4562-4e55-b3a1-340279f4d151 2024-09-18T00:23:34.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:23:34.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:23:34.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:23:34.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd2 2024-09-18T00:23:34.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:23:34.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd2 2024-09-18T00:23:34.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:23:34.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:23:34.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:23:34.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: ec224187-c7a4-4f71-81b4-2f03d6928b1f 2024-09-18T00:23:34.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:23:34.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:23:34.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:34.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:23:34.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:23:35.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:23:35.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:23:35.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:35.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary3 2024-09-18T00:23:35.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd2 test-primary3 2024-09-18T00:23:35.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:23:35.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary3 2024-09-18T00:23:35.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:23:35.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary3 2024-09-18T00:23:35.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:35.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:35.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:35.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary4 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary4 snapshot 10G 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:35.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary4 10G 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:35.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:23:35.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:23:35.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:23:35.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:35.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary4 2024-09-18T00:23:35.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary3 2024-09-18T00:23:35.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary3 2024-09-18T00:23:35.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary3 2024-09-18T00:23:35.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:35.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:23:35.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:35.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:35.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:23:35.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:23:35.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:23:35.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:35.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:23:35.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:35.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary4 snapshot 2024-09-18T00:23:35.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:35.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:35.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:35.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:35.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary4 snapshot 2024-09-18T00:23:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:23:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary3/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary3' 2024-09-18T00:23:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:23:36.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1329 2024-09-18T00:23:36.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary4 2024-09-18T00:23:36.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary4': 2024-09-18T00:23:36.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:23:36.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:23:36.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:23:36.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 22e97e81a7ed 2024-09-18T00:23:36.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.22e97e81a7ed 2024-09-18T00:23:36.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:23:36.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:23:36.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:23:36.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:23:36.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:23:35 2024 2024-09-18T00:23:36.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:23:35 2024 2024-09-18T00:23:36.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:23:35 2024 2024-09-18T00:23:36.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:23:36.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:23:36.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: f28d710c-fce9-48ab-b72f-a7c632f6c1ff 2024-09-18T00:23:36.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:23:36.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:23:36.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:23:36.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd3 2024-09-18T00:23:36.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:23:36.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd3 2024-09-18T00:23:36.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:23:36.995 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:23:36.994+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:23:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/26214402097152/2621440 done 2024-09-18T00:23:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:23:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 1c436f83-1ea1-4b9c-b1e6-a1a60ec9d74c 2024-09-18T00:23:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:23:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:23:37.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:37.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:23:37.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:23:37.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:23:37.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 Snapshot ID: 1330 2024-09-18T00:23:37.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:37.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:23:37.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:37.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary4 2024-09-18T00:23:37.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd3 test-primary4 2024-09-18T00:23:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:23:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:23:37.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:23:37.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary4 2024-09-18T00:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary4 2024-09-18T00:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary4 2024-09-18T00:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary5 2024-09-18T00:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary5 snapshot 10G 2024-09-18T00:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary4 2024-09-18T00:23:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary4 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary5 10G 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:37.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:37.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:23:37.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:23:37.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:23:37.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:37.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary5 2024-09-18T00:23:37.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:23:37.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:37.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:23:37.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:37.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary5 snapshot 2024-09-18T00:23:37.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:37.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:37.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:37.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:37.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary5 snapshot 2024-09-18T00:23:37.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:23:37.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary4/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary4' 2024-09-18T00:23:38.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1333 2024-09-18T00:23:38.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:23:38.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:38.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary5 2024-09-18T00:23:38.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary5': 2024-09-18T00:23:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:23:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:23:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:23:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 22fc94ca0945 2024-09-18T00:23:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.22fc94ca0945 2024-09-18T00:23:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:23:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:23:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:23:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:23:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:23:37 2024 2024-09-18T00:23:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:23:37 2024 2024-09-18T00:23:38.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:23:37 2024 2024-09-18T00:23:38.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:23:38.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:23:38.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 4fd082f9-a5cb-42cc-9a45-3ebe1a50bbca 2024-09-18T00:23:38.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:23:38.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:23:38.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:23:38.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd4 2024-09-18T00:23:38.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:23:38.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd4 2024-09-18T00:23:38.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:23:38.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:23:38.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:23:38.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: f21721f4-ae01-4483-ab20-094e8c71976c 2024-09-18T00:23:38.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:23:38.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:23:38.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:38.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:23:38.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:23:39.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:23:39.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:23:39.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:39.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary5 2024-09-18T00:23:39.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd4 test-primary5 2024-09-18T00:23:39.219 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:23:39.218+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:23:39.219 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:23:39.218+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:23:39.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:23:39.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary5 2024-09-18T00:23:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:23:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:23:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:23:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary6 2024-09-18T00:23:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary6 snapshot 10G 2024-09-18T00:23:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:39.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary6 10G 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:23:39.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:23:39.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:23:39.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:39.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary6 2024-09-18T00:23:39.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary5 2024-09-18T00:23:39.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary5 2024-09-18T00:23:39.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary5 2024-09-18T00:23:39.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary5 2024-09-18T00:23:39.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:39.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:23:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:39.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:23:39.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1332 2024-09-18T00:23:39.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:39.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:23:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:23:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary6 snapshot 2024-09-18T00:23:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:39.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary6 snapshot 2024-09-18T00:23:39.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:23:39.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary5/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary5' 2024-09-18T00:23:39.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:39.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:23:39.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:39.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:39.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:39.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:23:40.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:23:40.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1335 2024-09-18T00:23:40.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:40.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:23:40.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary6 2024-09-18T00:23:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary6': 2024-09-18T00:23:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:23:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 230ce97ce97d 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.230ce97ce97d 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:23:39 2024 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:23:39 2024 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:23:39 2024 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: b713c9d1-2c92-4cf8-9e0a-982f46d2d359 2024-09-18T00:23:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:23:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:23:40.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:23:40.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd5 2024-09-18T00:23:40.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:23:40.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd5 2024-09-18T00:23:40.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:23:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:23:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:23:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 29d81f37-b1f8-48d8-b41a-7396caea53cd 2024-09-18T00:23:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:23:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:23:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:40.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:23:40.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:23:41.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:23:41.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:23:41.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:41.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary6 2024-09-18T00:23:41.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd5 test-primary6 2024-09-18T00:23:41.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:23:41.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary6 2024-09-18T00:23:41.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary6 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary6 2024-09-18T00:23:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary7 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary7 snapshot 10G 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary6 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary6 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:23:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary7 10G 2024-09-18T00:23:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:41.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:41.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:23:41.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:23:41.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:23:41.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:41.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary7 2024-09-18T00:23:41.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:23:41.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:41.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:23:41.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:41.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary7 snapshot 2024-09-18T00:23:41.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:41.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:41.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:41.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:41.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary7 snapshot 2024-09-18T00:23:41.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:23:41.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary6/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary6' 2024-09-18T00:23:41.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1337 2024-09-18T00:23:41.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:41.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:41.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:23:41.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:41.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:41.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:41.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:23:41.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1339 2024-09-18T00:23:41.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:23:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:23:42.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1338 2024-09-18T00:23:42.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:42.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:23:42.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary7 2024-09-18T00:23:42.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary7': 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2322cb7a104 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2322cb7a104 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:23:41 2024 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:23:41 2024 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:23:41 2024 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:23:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:23:42.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 068a799c-59b8-4a9a-acb8-c91e7357896b 2024-09-18T00:23:42.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:23:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:23:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:23:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd6 2024-09-18T00:23:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:23:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd6 2024-09-18T00:23:42.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:23:42.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:23:42.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:23:42.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 90d1dff8-73f3-4dcd-a155-184470ce30f1 2024-09-18T00:23:42.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:23:42.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:23:42.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:42.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:23:42.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:23:43.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:23:43.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:23:43.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:43.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary7 2024-09-18T00:23:43.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd6 test-primary7 2024-09-18T00:23:43.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1341 2024-09-18T00:23:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:43.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:43.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:23:43.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:43.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:43.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:43.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:23:43.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:23:43.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary7 2024-09-18T00:23:43.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:23:43.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary7 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary7 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:23:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:23:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary8 2024-09-18T00:23:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary8 snapshot 10G 2024-09-18T00:23:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary7 2024-09-18T00:23:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary7 2024-09-18T00:23:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:23:43.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:43.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary8 10G 2024-09-18T00:23:43.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:43.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:43.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:43.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:43.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:43.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:43.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:23:43.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:23:43.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:23:43.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:43.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary8 2024-09-18T00:23:43.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:23:43.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary7/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary7' 2024-09-18T00:23:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:23:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:23:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary8 snapshot 2024-09-18T00:23:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary8 snapshot 2024-09-18T00:23:44.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1342 2024-09-18T00:23:44.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:44.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:23:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:23:44.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1344 2024-09-18T00:23:44.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:44.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:44.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:23:44.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:44.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:44.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:44.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:23:44.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:23:44.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary8 2024-09-18T00:23:44.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary8': 2024-09-18T00:23:44.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:23:44.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:23:44.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:23:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2335ed736ab 2024-09-18T00:23:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2335ed736ab 2024-09-18T00:23:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:23:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:23:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:23:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:23:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:23:43 2024 2024-09-18T00:23:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:23:43 2024 2024-09-18T00:23:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:23:43 2024 2024-09-18T00:23:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:23:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:23:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 51c15641-cc65-4aa0-a9cf-9e4d6425d92c 2024-09-18T00:23:44.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:23:44.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:23:44.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:23:44.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd7 2024-09-18T00:23:44.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:23:44.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd7 2024-09-18T00:23:44.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:23:44.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:23:44.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:23:44.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: ad7fb280-2811-4f90-bdcc-9a386dece865 2024-09-18T00:23:44.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:23:44.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:23:44.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:44.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:23:44.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:23:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:23:45.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:23:45.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:45.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary8 2024-09-18T00:23:45.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd7 test-primary8 2024-09-18T00:23:45.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:23:45.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary8 2024-09-18T00:23:45.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:23:45.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary8 2024-09-18T00:23:45.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary8 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary9 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary9 snapshot 10G 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary8 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary8 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary9 10G 2024-09-18T00:23:45.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:23:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:23:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:23:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary9 2024-09-18T00:23:45.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1343 2024-09-18T00:23:45.433 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:23:45.434+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:23:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:23:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:23:45.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:23:45.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:23:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:23:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary9 snapshot 2024-09-18T00:23:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary9 snapshot 2024-09-18T00:23:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:23:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary8/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary8' 2024-09-18T00:23:45.518 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:23:45.518+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:23:45.518 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:23:45.518+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:23:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:23:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:46.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:23:46.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1347 2024-09-18T00:23:46.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1346 2024-09-18T00:23:46.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:46.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:46.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1348 2024-09-18T00:23:46.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:23:46.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:46.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary9 2024-09-18T00:23:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary9': 2024-09-18T00:23:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:23:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:23:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:23:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 234c9a8300bd 2024-09-18T00:23:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.234c9a8300bd 2024-09-18T00:23:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:23:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:23:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:23:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:23:46.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:23:45 2024 2024-09-18T00:23:46.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:23:45 2024 2024-09-18T00:23:46.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:23:45 2024 2024-09-18T00:23:46.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:23:46.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:23:46.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: a4a49402-ee8f-4527-b9d9-e3c5d7289204 2024-09-18T00:23:46.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:23:46.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:23:46.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:23:46.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd8 2024-09-18T00:23:46.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:23:46.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd8 2024-09-18T00:23:46.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:23:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:23:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:23:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: d456ae4f-6aa4-4658-9eba-5434a26996af 2024-09-18T00:23:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:23:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:23:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:23:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:23:47.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:23:47.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:23:47.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:47.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary9 2024-09-18T00:23:47.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd8 test-primary9 2024-09-18T00:23:47.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1349 2024-09-18T00:23:47.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:47.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:23:47.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary9 2024-09-18T00:23:47.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:23:47.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary9 2024-09-18T00:23:47.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:47.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:23:47.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:47.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:47.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:23:47.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary9 2024-09-18T00:23:47.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:47.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:23:47.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary10 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary10 snapshot 10G 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary9 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary9 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:47.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary10 10G 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:23:47.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:23:47.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary10 2024-09-18T00:23:47.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:47.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:23:47.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:47.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:47.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:47.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:23:47.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:23:47.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:47.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:23:47.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:23:47.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary10 snapshot 2024-09-18T00:23:47.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:47.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:47.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:47.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:23:47.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary10 snapshot 2024-09-18T00:23:47.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:47.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:23:47.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:47.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:47.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:47.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:23:47.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:23:47.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary9/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary9' 2024-09-18T00:23:48.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:48.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:23:48.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:48.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:48.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:48.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:23:48.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1351 2024-09-18T00:23:48.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:48.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:23:48.549 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:23:48.550+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d674448c30 2024-09-18T00:23:48.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1354 2024-09-18T00:23:48.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary10 2024-09-18T00:23:48.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:48.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary10': 2024-09-18T00:23:48.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:23:48.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:23:48.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 23608f7eb66c 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.23608f7eb66c 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:23:47 2024 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:23:47 2024 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:23:47 2024 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 4ce86fed-82d5-428f-affa-37bf7b004ee8 2024-09-18T00:23:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:23:48.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:23:48.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:23:48.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd9 2024-09-18T00:23:48.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:23:48.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd9 2024-09-18T00:23:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:23:48.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:23:48.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:23:48.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 06b878b1-b45f-4259-a981-56f3a011b49c 2024-09-18T00:23:48.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:23:48.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:23:48.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:48.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:23:48.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:23:49.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:23:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:23:49.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:23:49.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1355 2024-09-18T00:23:49.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary10 2024-09-18T00:23:49.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd9 test-primary10 2024-09-18T00:23:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:49.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1352 2024-09-18T00:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:23:49.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:49.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:23:49.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:23:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:49.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:23:49.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:23:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:23:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:49.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:23:49.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary10 2024-09-18T00:23:49.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:23:49.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:23:49.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:23:49.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 74928 2024-09-18T00:23:49.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary10 2024-09-18T00:23:49.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary10 2024-09-18T00:23:49.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary10 2024-09-18T00:23:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary10 2024-09-18T00:23:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:23:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:23:49.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:23:49.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary10/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary10' 2024-09-18T00:23:50.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:50.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:23:50.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:50.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:50.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:23:50.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:23:50.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1356 2024-09-18T00:23:50.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:50.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1360 2024-09-18T00:23:50.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:51.246 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:23:51.243+0000 7f52d7c6c640 -1 reset not still connected to 0x5578426955f0 2024-09-18T00:23:51.246 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:23:51.243+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T00:23:51.246 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:23:51.243+0000 7f52d7c6c640 -1 reset not still connected to 0x557843021860 2024-09-18T00:23:51.246 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:23:51.243+0000 7f52d7c6c640 -1 reset not still connected to 0x557843079e10 2024-09-18T00:23:51.246 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:23:51.243+0000 7f52d7c6c640 -1 reset not still connected to 0x55784372dd40 2024-09-18T00:23:51.246 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:23:51.243+0000 7f52d7c6c640 -1 reset not still connected to 0x5578446055f0 2024-09-18T00:23:51.246 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:23:51.243+0000 7f52d7c6c640 -1 reset not still connected to 0x557844a984e0 2024-09-18T00:23:51.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:51.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:23:51.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:51.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:51.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:51.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:23:51.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1359 2024-09-18T00:23:51.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1357 2024-09-18T00:23:51.531 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:23:51.530+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d677845e10 2024-09-18T00:23:51.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:51.552 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:23:51.542+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d67f953ba0 2024-09-18T00:23:51.553 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:23:51.542+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d69f573450 2024-09-18T00:23:51.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:51.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1358 2024-09-18T00:23:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:23:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:51.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:23:51.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:52.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1361 2024-09-18T00:23:52.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:52.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:52.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:23:52.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:52.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:52.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:52.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:23:52.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:52.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:23:52.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:52.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:52.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:52.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:23:53.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:53.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:23:53.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:53.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:53.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:53.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:23:53.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1363 2024-09-18T00:23:53.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1362 2024-09-18T00:23:53.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:53.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:23:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:53.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:23:54.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1365 2024-09-18T00:23:54.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1364 2024-09-18T00:23:54.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:54.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:23:54.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:54.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:54.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:54.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:23:54.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:54.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:54.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:54.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:23:54.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:54.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:54.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:54.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:23:54.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:54.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:23:54.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:54.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:54.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:54.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:23:55.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:55.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:23:55.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:55.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:55.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:23:55.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:23:55.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1367 2024-09-18T00:23:55.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1366 2024-09-18T00:23:55.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:56.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1369 2024-09-18T00:23:56.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1370 2024-09-18T00:23:56.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:23:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:23:56.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:23:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:23:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:23:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:23:56.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1368 2024-09-18T00:23:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:56.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:56.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:57.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1371 2024-09-18T00:23:57.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:23:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:23:57.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:23:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:23:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:23:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:23:58.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:58.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:23:58.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:58.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:58.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:23:58.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:23:58.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1372 2024-09-18T00:23:58.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1373 2024-09-18T00:23:58.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:58.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:23:58.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:58.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:58.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:23:58.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:23:58.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:58.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:59.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:59.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:23:59.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:59.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:59.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:23:59.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:23:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:23:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:23:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:23:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1375 2024-09-18T00:23:59.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1374 2024-09-18T00:23:59.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:23:59.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:23:59.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:23:59.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:23:59.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:23:59.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:23:59.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:23:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:00.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1376 2024-09-18T00:24:00.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:00.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:24:00.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:00.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:00.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:24:00.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:24:00.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1377 2024-09-18T00:24:00.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:00.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:01.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:01.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:24:01.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:01.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:01.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:24:01.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:24:01.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:01.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:24:01.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:01.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:01.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:24:01.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:24:01.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1378 2024-09-18T00:24:01.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:01.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1380 2024-09-18T00:24:01.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1379 2024-09-18T00:24:01.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:01.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:02.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1381 2024-09-18T00:24:02.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:02.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:02.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:24:02.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:02.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:02.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:24:02.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:24:02.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:02.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:24:02.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:02.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:02.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:24:02.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:24:03.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:24:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:24:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:24:03.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:03.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:24:03.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:03.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:03.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:24:03.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:24:03.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1383 2024-09-18T00:24:03.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1382 2024-09-18T00:24:03.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:04.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1385 2024-09-18T00:24:04.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1384 2024-09-18T00:24:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:24:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:04.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:24:04.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:24:04.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:04.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:04.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:04.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:24:04.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:24:04.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:04.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:24:04.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:04.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:04.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:24:04.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:24:05.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:05.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:24:05.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:05.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:05.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:24:05.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:24:05.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1386 2024-09-18T00:24:05.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1387 2024-09-18T00:24:05.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:05.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:06.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1388 2024-09-18T00:24:06.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1389 2024-09-18T00:24:06.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1390 2024-09-18T00:24:06.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:06.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:24:06.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:06.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:06.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:24:06.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:24:06.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:06.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:24:06.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:06.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:06.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:24:06.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:24:06.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:06.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:06.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:07.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1391 2024-09-18T00:24:07.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:24:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:24:07.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:24:07.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:07.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:24:07.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:07.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:07.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:24:07.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:24:08.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1392 2024-09-18T00:24:08.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:08.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1393 2024-09-18T00:24:08.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:08.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:08.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:24:08.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:08.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:08.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:24:08.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:24:08.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:08.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:24:08.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:08.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:08.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:24:08.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:24:09.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1394 2024-09-18T00:24:09.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:09.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1395 2024-09-18T00:24:09.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:09.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:09.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:24:09.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:09.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:09.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:24:09.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:24:09.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:09.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:24:09.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:09.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:09.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:24:09.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:24:09.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:09.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:24:09.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:09.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:09.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:24:09.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:24:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1397 2024-09-18T00:24:10.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:10.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:24:10.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:10.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:10.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:24:10.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:24:10.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1396 2024-09-18T00:24:10.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:10.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:11.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:11.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:24:11.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:11.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:11.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:24:11.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:24:11.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1398 2024-09-18T00:24:11.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1400 2024-09-18T00:24:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:24:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:24:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:24:11.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1399 2024-09-18T00:24:11.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:11.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:12.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1401 2024-09-18T00:24:12.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:12.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:12.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:24:12.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:12.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:12.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:24:12.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:24:12.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:12.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:24:12.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:12.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:12.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:24:12.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:24:13.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:13.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:24:13.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:13.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:13.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:24:13.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:24:13.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1402 2024-09-18T00:24:13.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1403 2024-09-18T00:24:13.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:13.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:24:13.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:13.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:13.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:24:13.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:24:13.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:13.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:14.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1404 2024-09-18T00:24:14.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1405 2024-09-18T00:24:14.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:14.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:14.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:14.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:24:14.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:14.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:14.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:24:14.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:24:14.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:14.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:24:14.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:14.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:14.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:24:14.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:24:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:24:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:24:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:24:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:24:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:24:15.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:24:15.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1406 2024-09-18T00:24:15.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:15.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1407 2024-09-18T00:24:15.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:16.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1408 2024-09-18T00:24:16.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1410 2024-09-18T00:24:16.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:16.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:24:16.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:16.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:16.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:24:16.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:24:16.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:16.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:24:16.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:16.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:16.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:24:16.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:24:16.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:16.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1409 2024-09-18T00:24:16.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:17.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1411 2024-09-18T00:24:17.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:17.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:17.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:24:17.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:17.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:17.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:24:17.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:24:17.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:17.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:24:17.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:17.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:17.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:24:17.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:24:18.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:18.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:24:18.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:18.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:18.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:24:18.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:24:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1413 2024-09-18T00:24:18.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:18.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:24:18.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:18.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:18.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:24:18.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:24:18.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1412 2024-09-18T00:24:18.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:18.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:19.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:19.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:24:19.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:19.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:19.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:24:19.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:24:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1415 2024-09-18T00:24:19.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:19.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:24:19.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:19.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:19.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:24:19.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:24:19.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1414 2024-09-18T00:24:19.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:19.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:24:19.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:19.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:19.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:24:19.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:24:19.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:19.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:20.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:24:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:24:20.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:24:20.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1416 2024-09-18T00:24:20.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:20.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1417 2024-09-18T00:24:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:21.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1418 2024-09-18T00:24:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:24:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:24:21.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:24:21.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:21.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:24:21.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:21.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:21.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:24:21.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:24:21.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:21.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1419 2024-09-18T00:24:21.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:22.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1421 2024-09-18T00:24:22.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:22.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1422 2024-09-18T00:24:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:24:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:24:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:24:22.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:22.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:24:22.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:22.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:22.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:24:22.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:24:23.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:23.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:24:23.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:23.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:23.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:24:23.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:24:23.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1424 2024-09-18T00:24:23.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:23.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:23.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:24:23.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:23.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:23.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:24:23.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:24:23.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1423 2024-09-18T00:24:23.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:24.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1426 2024-09-18T00:24:24.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1425 2024-09-18T00:24:24.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:24:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:24:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:24:24.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:24:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:24.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:24.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:24:24.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:24:25.167 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:24:25.165+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T00:24:25.168 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:24:25.165+0000 7f52d7c6c640 -1 reset not still connected to 0x557843021860 2024-09-18T00:24:25.169 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:24:25.165+0000 7f52d7c6c640 -1 reset not still connected to 0x557843079e10 2024-09-18T00:24:25.169 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:24:25.165+0000 7f52d7c6c640 -1 reset not still connected to 0x55784372dd40 2024-09-18T00:24:25.169 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:24:25.165+0000 7f52d7c6c640 -1 reset not still connected to 0x5578446055f0 2024-09-18T00:24:25.169 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:24:25.165+0000 7f52d7c6c640 -1 reset not still connected to 0x557844a984e0 2024-09-18T00:24:25.169 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:24:25.165+0000 7f52d7c6c640 -1 reset not still connected to 0x5578453d2270 2024-09-18T00:24:25.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:25.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:24:25.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:25.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:25.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:24:25.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:24:25.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:25.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:24:25.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:25.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:25.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:24:25.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:24:25.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1427 2024-09-18T00:24:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1428 2024-09-18T00:24:25.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:25.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:26.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1429 2024-09-18T00:24:26.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:26.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:24:26.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:26.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:26.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:24:26.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:24:26.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:26.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1430 2024-09-18T00:24:26.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:26.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:26.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:24:26.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:26.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:26.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:24:26.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:24:27.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1432 2024-09-18T00:24:27.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1431 2024-09-18T00:24:27.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:27.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:27.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:27.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:24:27.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:27.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:27.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:24:27.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:24:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:24:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:24:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:24:28.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1433 2024-09-18T00:24:28.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:24:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:24:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:24:28.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1434 2024-09-18T00:24:28.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:28.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:24:28.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:28.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:28.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:24:28.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:24:28.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1436 2024-09-18T00:24:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:24:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:24:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:24:29.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1435 2024-09-18T00:24:29.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:29.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:24:29.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:29.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:29.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:24:29.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:24:29.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:29.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:30.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:30.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:24:30.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:30.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:30.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:24:30.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:24:30.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:30.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:24:30.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:30.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:30.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:24:30.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:24:30.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1437 2024-09-18T00:24:30.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1438 2024-09-18T00:24:30.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:30.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:24:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:31.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:24:31.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:24:31.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1439 2024-09-18T00:24:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:31.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1440 2024-09-18T00:24:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:24:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:24:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:24:31.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:32.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1442 2024-09-18T00:24:32.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1441 2024-09-18T00:24:32.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:32.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:32.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:32.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:24:32.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:32.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:32.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:24:32.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:24:32.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:32.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:24:32.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:32.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:32.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:24:32.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:24:33.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1444 2024-09-18T00:24:33.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1443 2024-09-18T00:24:33.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:24:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:24:33.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:24:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:24:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:24:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:24:33.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:34.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:34.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:24:34.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:34.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:34.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:24:34.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:24:34.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:34.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:24:34.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:34.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:34.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:24:34.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:24:34.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1445 2024-09-18T00:24:34.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1446 2024-09-18T00:24:34.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:34.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:35.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:35.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:24:35.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:35.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:35.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:24:35.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:24:35.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:35.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:24:35.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:35.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:35.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:24:35.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:24:35.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1448 2024-09-18T00:24:35.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1447 2024-09-18T00:24:35.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:35.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:36.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1449 2024-09-18T00:24:36.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:36.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:24:36.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:36.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:36.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:24:36.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:24:36.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1450 2024-09-18T00:24:36.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:36.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:24:36.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:36.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:36.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:24:36.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:24:36.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:36.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1451 2024-09-18T00:24:37.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1452 2024-09-18T00:24:37.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:37.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:37.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:37.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:24:37.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:37.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:37.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:24:37.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:24:37.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:37.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:24:37.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:37.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:37.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:24:37.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:24:38.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1454 2024-09-18T00:24:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:24:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:38.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:24:38.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:24:38.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1453 2024-09-18T00:24:38.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:24:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:24:38.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1455 2024-09-18T00:24:39.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1456 2024-09-18T00:24:39.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:39.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:24:39.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:39.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:39.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:24:39.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:24:39.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:39.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:39.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:24:39.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:39.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:39.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:24:39.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:24:39.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:40.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:40.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:24:40.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:40.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:40.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:24:40.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:24:40.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:40.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:24:40.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:40.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:40.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:24:40.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:24:40.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1458 2024-09-18T00:24:40.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1457 2024-09-18T00:24:40.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:40.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:41.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:24:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:24:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:24:41.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1460 2024-09-18T00:24:41.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1459 2024-09-18T00:24:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:24:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:24:41.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:24:41.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:42.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1461 2024-09-18T00:24:42.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1462 2024-09-18T00:24:42.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:42.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:42.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:42.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:24:42.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:42.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:42.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:24:42.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:24:42.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:42.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:24:42.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:42.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:42.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:24:42.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:24:43.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1464 2024-09-18T00:24:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:24:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:24:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:24:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:24:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:43.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:24:43.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:24:43.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1463 2024-09-18T00:24:43.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:43.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:44.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1465 2024-09-18T00:24:44.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1466 2024-09-18T00:24:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:24:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:24:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:24:44.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:44.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:24:44.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:44.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:44.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:24:44.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:24:44.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:44.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:45.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:45.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:24:45.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:45.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:45.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:24:45.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:24:45.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:45.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:24:45.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:45.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:45.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:24:45.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:24:45.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1468 2024-09-18T00:24:45.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1467 2024-09-18T00:24:45.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:46.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:46.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:24:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:24:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:24:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1470 2024-09-18T00:24:46.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1469 2024-09-18T00:24:46.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:24:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:24:46.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:24:46.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:47.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1472 2024-09-18T00:24:47.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1471 2024-09-18T00:24:47.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:47.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:47.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:47.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:24:47.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:47.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:47.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:24:47.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:24:47.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:47.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:24:47.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:47.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:47.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:24:47.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:24:48.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1473 2024-09-18T00:24:48.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1474 2024-09-18T00:24:48.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:48.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:24:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:48.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:48.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:24:48.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:24:48.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:48.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:24:49.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1476 2024-09-18T00:24:49.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:49.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1475 2024-09-18T00:24:49.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:49.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:49.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:24:49.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:49.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:49.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:24:49.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:24:49.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:49.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:24:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:24:49.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:24:50.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1478 2024-09-18T00:24:50.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:50.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:24:50.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:50.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:50.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:24:50.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:24:50.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:50.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:24:50.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:50.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:50.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:24:50.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:24:50.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:50.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1477 2024-09-18T00:24:50.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:51.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1480 2024-09-18T00:24:51.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:51.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:24:51.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:51.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:51.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:24:51.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:24:51.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:51.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:24:51.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:51.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:51.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:24:51.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:24:51.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1479 2024-09-18T00:24:51.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:51.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:52.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1481 2024-09-18T00:24:52.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:24:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:24:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:24:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:52.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:52.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:24:52.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:52.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:52.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:24:52.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:24:52.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1482 2024-09-18T00:24:52.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:53.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:53.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:24:53.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:53.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:53.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:24:53.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:24:53.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:53.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:24:53.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:53.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:53.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:24:53.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:24:53.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1483 2024-09-18T00:24:53.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1484 2024-09-18T00:24:53.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:54.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1485 2024-09-18T00:24:54.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1486 2024-09-18T00:24:54.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:54.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:24:54.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:54.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:54.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:24:54.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:24:54.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:54.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:24:54.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:54.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:54.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:24:54.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:24:54.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:54.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:55.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:55.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:24:55.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:55.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:55.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:24:55.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:24:55.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1487 2024-09-18T00:24:55.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:55.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:24:55.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:55.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:55.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:24:55.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:24:55.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1488 2024-09-18T00:24:55.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:56.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1489 2024-09-18T00:24:56.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1490 2024-09-18T00:24:56.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:56.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:56.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:56.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:24:56.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:24:56.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:24:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:24:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:24:56.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:24:57.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1492 2024-09-18T00:24:57.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1491 2024-09-18T00:24:57.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:57.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:57.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:24:57.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:57.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:57.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:24:57.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:24:57.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:57.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:24:57.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:57.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:57.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:24:57.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:24:58.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1493 2024-09-18T00:24:58.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:58.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:24:58.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:58.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:58.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:24:58.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:24:58.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:58.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1494 2024-09-18T00:24:58.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:58.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:24:58.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:58.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:58.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:24:58.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:24:58.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:59.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:59.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:24:59.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:59.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:59.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:24:59.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:24:59.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:24:59.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:24:59.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:24:59.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:24:59.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:24:59.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:24:59.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1495 2024-09-18T00:24:59.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1496 2024-09-18T00:24:59.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:24:59.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:00.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:00.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:25:00.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:00.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:00.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:25:00.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:25:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:25:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:25:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:25:00.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1497 2024-09-18T00:25:00.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1498 2024-09-18T00:25:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:00.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:01.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1499 2024-09-18T00:25:01.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1500 2024-09-18T00:25:01.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:01.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:25:01.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:01.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:01.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:25:01.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:25:01.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:01.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:01.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:01.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:25:01.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:01.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:01.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:25:01.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:25:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1501 2024-09-18T00:25:02.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1502 2024-09-18T00:25:02.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:02.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:02.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:02.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:25:02.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:02.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:02.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:25:02.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:25:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:25:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:25:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:25:03.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1503 2024-09-18T00:25:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:25:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:03.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:03.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:25:03.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:25:03.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:03.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:03.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:25:03.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:03.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:03.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:25:03.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:25:03.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1504 2024-09-18T00:25:03.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:04.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:04.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:25:04.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:04.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:04.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:25:04.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:25:04.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:25:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:25:04.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:25:04.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1505 2024-09-18T00:25:04.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1506 2024-09-18T00:25:04.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:04.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:25:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:25:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:25:05.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:05.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:25:05.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:05.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:05.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:25:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:25:05.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1508 2024-09-18T00:25:05.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1507 2024-09-18T00:25:05.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:05.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:25:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:25:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:25:06.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:06.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:25:06.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:06.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:06.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:25:06.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:25:06.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1509 2024-09-18T00:25:06.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:06.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1510 2024-09-18T00:25:06.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:07.481 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:25:07.479+0000 7f849c00d640 -1 reset not still connected to 0x5629a760dc70 2024-09-18T00:25:07.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1512 2024-09-18T00:25:07.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1511 2024-09-18T00:25:07.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:07.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:07.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:07.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:25:07.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:07.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:07.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:25:07.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:25:07.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:07.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:25:07.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:07.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:07.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:25:07.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:25:08.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1514 2024-09-18T00:25:08.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1513 2024-09-18T00:25:08.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:08.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:25:08.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:08.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:08.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:25:08.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:25:08.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:25:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:25:08.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:25:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:09.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1515 2024-09-18T00:25:09.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1516 2024-09-18T00:25:09.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:09.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:09.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:09.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:25:09.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:09.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:09.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:25:09.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:25:09.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:25:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:25:09.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:25:10.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:10.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:25:10.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:10.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:10.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:25:10.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:25:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:25:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:10.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:25:10.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:25:10.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1518 2024-09-18T00:25:10.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1517 2024-09-18T00:25:10.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:10.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:11.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1519 2024-09-18T00:25:11.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:25:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:25:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:25:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:25:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:25:11.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:25:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1520 2024-09-18T00:25:11.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:12.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1522 2024-09-18T00:25:12.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1521 2024-09-18T00:25:12.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:12.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:12.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:12.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:25:12.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:12.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:12.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:25:12.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:25:12.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:12.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:25:12.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:12.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:12.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:25:12.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:25:13.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1523 2024-09-18T00:25:13.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:13.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:25:13.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:13.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:13.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:25:13.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:25:13.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:13.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1524 2024-09-18T00:25:13.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:13.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:25:13.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:13.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:13.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:25:13.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:25:13.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:14.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:14.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:25:14.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:14.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:14.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:25:14.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:25:14.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1525 2024-09-18T00:25:14.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1526 2024-09-18T00:25:14.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:14.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:25:14.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:14.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:14.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:25:14.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:25:14.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:14.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:15.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:15.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:25:15.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:15.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:15.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:25:15.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:25:15.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:15.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:25:15.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:15.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:15.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:25:15.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:25:15.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1528 2024-09-18T00:25:15.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:15.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1527 2024-09-18T00:25:15.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:16.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1529 2024-09-18T00:25:16.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:16.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:25:16.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:16.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:16.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:25:16.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:25:16.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1530 2024-09-18T00:25:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:16.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:16.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:16.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:25:16.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:16.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:16.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:25:16.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:25:17.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1531 2024-09-18T00:25:17.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1532 2024-09-18T00:25:17.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:17.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:17.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:25:17.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:17.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:17.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:25:17.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:25:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:25:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:25:17.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:25:18.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1533 2024-09-18T00:25:18.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1534 2024-09-18T00:25:18.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:18.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:18.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:25:18.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:18.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:18.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:25:18.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:25:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:25:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:25:18.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:25:19.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1535 2024-09-18T00:25:19.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1536 2024-09-18T00:25:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:19.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:19.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:25:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:25:19.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:25:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:25:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:25:19.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:25:20.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1537 2024-09-18T00:25:20.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:20.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:25:20.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:20.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:20.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:25:20.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:25:20.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1538 2024-09-18T00:25:20.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:20.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:25:20.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:20.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:20.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:25:20.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:25:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:21.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1539 2024-09-18T00:25:21.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:21.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:21.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:25:21.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:21.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:21.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:25:21.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1540 2024-09-18T00:25:21.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:25:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:25:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:21.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:21.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:25:21.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:25:21.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:22.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1542 2024-09-18T00:25:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:25:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:25:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:25:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:25:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:25:22.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:25:22.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:22.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1541 2024-09-18T00:25:22.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:23.237 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:25:23.232+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a340 2024-09-18T00:25:23.238 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:25:23.232+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T00:25:23.238 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:25:23.232+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T00:25:23.239 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:25:23.240+0000 7fd4c158b640 -1 reset not still connected to 0x564369004000 2024-09-18T00:25:23.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1544 2024-09-18T00:25:23.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1543 2024-09-18T00:25:23.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:23.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:25:23.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:23.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:23.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:23.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:25:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:25:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:25:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:25:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:25:23.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:23.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1546 2024-09-18T00:25:24.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1545 2024-09-18T00:25:24.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:24.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:25:24.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:24.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:24.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:25:24.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:25:24.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:24.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:24.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:25:24.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:24.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:24.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:25:24.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:25:24.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:25.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1548 2024-09-18T00:25:25.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:25.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:25:25.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:25.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:25.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:25:25.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:25:25.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:25.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1547 2024-09-18T00:25:25.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:25.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:25:25.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:25.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:25.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:25:25.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:25:25.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:26.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:26.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:25:26.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:26.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:26.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:25:26.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:25:26.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1549 2024-09-18T00:25:26.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:26.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:25:26.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:26.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:26.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:25:26.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:25:26.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1550 2024-09-18T00:25:26.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:26.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:27.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1551 2024-09-18T00:25:27.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:27.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:27.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:25:27.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:27.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:27.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:25:27.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:25:27.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1552 2024-09-18T00:25:27.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:27.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:25:27.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:27.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:27.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:25:27.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:25:27.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:28.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1554 2024-09-18T00:25:28.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1553 2024-09-18T00:25:28.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:28.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:28.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:25:28.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:28.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:28.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:25:28.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:25:28.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:28.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:28.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:25:28.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:28.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:28.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:25:28.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:25:29.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1555 2024-09-18T00:25:29.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1556 2024-09-18T00:25:29.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:29.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:29.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:25:29.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:29.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:29.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:25:29.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:25:29.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:25:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:25:29.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:25:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:25:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:25:30.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:25:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:25:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:25:30.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:25:30.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1557 2024-09-18T00:25:30.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:30.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1558 2024-09-18T00:25:30.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:31.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1559 2024-09-18T00:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:25:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:25:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:25:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:25:31.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:31.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1560 2024-09-18T00:25:31.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:32.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1562 2024-09-18T00:25:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1561 2024-09-18T00:25:32.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:32.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:32.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:32.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:25:32.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:32.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:32.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:25:32.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:25:32.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:32.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:25:32.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:32.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:32.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:25:32.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:25:33.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1564 2024-09-18T00:25:33.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1563 2024-09-18T00:25:33.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:33.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:25:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:25:33.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:25:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:25:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:33.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:25:33.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:25:34.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1566 2024-09-18T00:25:34.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:34.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:25:34.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:34.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:34.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:25:34.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:25:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1565 2024-09-18T00:25:34.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:34.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:25:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:25:34.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:25:35.147 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:25:35.148+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T00:25:35.155 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:25:35.152+0000 7f52d7c6c640 -1 reset not still connected to 0x557843079e10 2024-09-18T00:25:35.155 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:25:35.152+0000 7f52d7c6c640 -1 reset not still connected to 0x55784372dd40 2024-09-18T00:25:35.155 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:25:35.152+0000 7f52d7c6c640 -1 reset not still connected to 0x5578446055f0 2024-09-18T00:25:35.155 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:25:35.152+0000 7f52d7c6c640 -1 reset not still connected to 0x557844a984e0 2024-09-18T00:25:35.155 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:25:35.152+0000 7f52d7c6c640 -1 reset not still connected to 0x5578453d2270 2024-09-18T00:25:35.155 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:25:35.152+0000 7f52d7c6c640 -1 reset not still connected to 0x5578476c3380 2024-09-18T00:25:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:25:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:25:35.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:25:35.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:35.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:25:35.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:35.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:35.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:25:35.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:25:35.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1567 2024-09-18T00:25:35.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1568 2024-09-18T00:25:35.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:35.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:36.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:36.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:25:36.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:36.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:36.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:25:36.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:25:36.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:36.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:25:36.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:36.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:36.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:25:36.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:25:36.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1570 2024-09-18T00:25:36.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1569 2024-09-18T00:25:36.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:36.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:37.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1572 2024-09-18T00:25:37.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1571 2024-09-18T00:25:37.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:37.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:37.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:37.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:25:37.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:37.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:37.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:25:37.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:25:37.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:37.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:25:37.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:37.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:37.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:25:37.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:25:38.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:38.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:25:38.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:38.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:38.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:25:38.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:25:38.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1574 2024-09-18T00:25:38.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:38.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:25:38.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:38.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:38.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:25:38.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:25:38.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:38.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1573 2024-09-18T00:25:38.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:39.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1576 2024-09-18T00:25:39.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:25:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:25:39.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:25:39.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1575 2024-09-18T00:25:39.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:39.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:39.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:25:39.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:39.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:39.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:25:39.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:25:39.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:25:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:25:40.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:25:40.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:25:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:25:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:25:40.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1577 2024-09-18T00:25:40.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1578 2024-09-18T00:25:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:40.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:41.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1580 2024-09-18T00:25:41.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:41.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:25:41.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:41.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:41.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:25:41.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:25:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1579 2024-09-18T00:25:41.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:41.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:25:41.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:41.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:41.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:25:41.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:25:41.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:42.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1581 2024-09-18T00:25:42.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1582 2024-09-18T00:25:42.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:42.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:42.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:42.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:25:42.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:42.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:42.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:25:42.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:25:43.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1583 2024-09-18T00:25:43.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1584 2024-09-18T00:25:43.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:43.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:43.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:43.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:25:43.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:43.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:43.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:25:43.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:25:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:25:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:43.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:25:43.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:25:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:25:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:25:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:25:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1586 2024-09-18T00:25:44.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1585 2024-09-18T00:25:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:25:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:25:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:25:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:45.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:45.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:25:45.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:45.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:45.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:25:45.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:25:45.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:45.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:25:45.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:45.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:45.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:25:45.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:25:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1587 2024-09-18T00:25:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1588 2024-09-18T00:25:45.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:45.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:46.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1589 2024-09-18T00:25:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:25:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:25:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:25:46.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:46.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:25:46.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:46.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:46.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:25:46.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:25:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:46.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1590 2024-09-18T00:25:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:46.678 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:25:46.677+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d674152820 2024-09-18T00:25:47.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1591 2024-09-18T00:25:47.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:47.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1592 2024-09-18T00:25:47.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:47.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:47.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:25:47.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:47.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:47.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:25:47.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:25:47.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:47.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:25:47.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:47.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:47.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:25:47.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:25:48.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1593 2024-09-18T00:25:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:25:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:25:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:25:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:25:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:25:48.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:25:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1594 2024-09-18T00:25:48.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:48.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:49.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:49.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:25:49.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:49.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:49.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:25:49.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:25:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1596 2024-09-18T00:25:49.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1595 2024-09-18T00:25:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:25:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:25:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:25:49.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:49.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:50.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:50.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:25:50.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:50.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:50.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:25:50.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:25:50.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:50.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:25:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:25:50.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:25:50.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1597 2024-09-18T00:25:50.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:50.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1598 2024-09-18T00:25:50.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:51.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1600 2024-09-18T00:25:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:25:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:25:51.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:25:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:25:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:25:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:25:51.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1599 2024-09-18T00:25:51.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:52.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1601 2024-09-18T00:25:52.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1602 2024-09-18T00:25:52.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:52.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:25:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:25:52.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:25:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:25:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:25:52.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:25:53.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1603 2024-09-18T00:25:53.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1604 2024-09-18T00:25:53.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:53.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:53.526 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:25:53.521+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735ae340 2024-09-18T00:25:53.526 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:25:53.521+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735aedd0 2024-09-18T00:25:53.527 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:25:53.525+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d676592410 2024-09-18T00:25:53.527 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:25:53.525+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d677845e10 2024-09-18T00:25:53.528 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:25:53.525+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d67e51c750 2024-09-18T00:25:53.528 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:25:53.525+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d67f953ba0 2024-09-18T00:25:53.528 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:25:53.525+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d69f573450 2024-09-18T00:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:53.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:25:53.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:25:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:25:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:25:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:25:54.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1606 2024-09-18T00:25:54.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1605 2024-09-18T00:25:54.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:54.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:54.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:54.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:25:54.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:54.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:54.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:25:54.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:25:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:25:54.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:54.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:54.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:25:54.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:25:55.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1607 2024-09-18T00:25:55.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1608 2024-09-18T00:25:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:25:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:25:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:25:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:25:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:25:55.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:25:55.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:55.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:56.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1609 2024-09-18T00:25:56.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1610 2024-09-18T00:25:56.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:56.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:25:56.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:56.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:56.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:25:56.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:25:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:25:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:25:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:25:56.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:56.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:57.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1612 2024-09-18T00:25:57.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1611 2024-09-18T00:25:57.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:57.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:57.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:57.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:25:57.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:57.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:57.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:25:57.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:25:57.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:57.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:25:57.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:57.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:57.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:25:57.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:25:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1613 2024-09-18T00:25:58.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:58.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:58.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:25:58.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:25:58.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:58.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:58.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:58.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:58.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:25:58.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:25:58.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:25:58.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:25:58.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1614 2024-09-18T00:25:58.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:58.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:25:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:25:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:25:59.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:25:59.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:25:59.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:25:59.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:25:59.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:25:59.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:25:59.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1616 2024-09-18T00:25:59.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1615 2024-09-18T00:25:59.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:25:59.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:00.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:26:00.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 75062 2024-09-18T00:26:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:26:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:26:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:26:00.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1618 2024-09-18T00:26:00.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1617 2024-09-18T00:26:00.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:00.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:01.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1620 2024-09-18T00:26:01.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:01.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:26:01.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:01.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:01.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:26:01.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:26:01.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:01.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:26:01.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:01.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:01.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:26:01.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:26:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1619 2024-09-18T00:26:01.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:01.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:02.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1621 2024-09-18T00:26:02.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:02.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:26:02.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 75200 2024-09-18T00:26:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:26:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:02.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:26:02.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:26:03.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1623 2024-09-18T00:26:03.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:03.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:26:03.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:03.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:03.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:26:03.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:26:03.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1622 2024-09-18T00:26:03.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:03.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:26:03.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:03.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:03.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:26:03.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:26:03.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:03.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:04.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1624 2024-09-18T00:26:04.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:26:04.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:04.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:04.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:26:04.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:26:04.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:04.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:26:04.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:04.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:04.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:26:04.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:26:05.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:26:05.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 75372 2024-09-18T00:26:05.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1626 2024-09-18T00:26:05.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1625 2024-09-18T00:26:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:05.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:06.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1627 2024-09-18T00:26:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1628 2024-09-18T00:26:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:26:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:26:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:26:06.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:06.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:26:06.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 75520 2024-09-18T00:26:06.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:07.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:07.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:26:07.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:07.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:07.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:26:07.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:26:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1629 2024-09-18T00:26:08.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:26:08.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 75694 2024-09-18T00:26:08.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:26:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:26:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:26:09.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1630 2024-09-18T00:26:09.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:26:09.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 75893 2024-09-18T00:26:09.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:09.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:09.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:26:09.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:09.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:09.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:26:09.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:26:10.117 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:26:10.115+0000 7f52d7c6c640 -1 reset not still connected to 0x557843079e10 2024-09-18T00:26:10.128 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:26:10.127+0000 7f52d7c6c640 -1 reset not still connected to 0x55784372dd40 2024-09-18T00:26:10.128 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:26:10.127+0000 7f52d7c6c640 -1 reset not still connected to 0x5578446055f0 2024-09-18T00:26:10.128 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:26:10.127+0000 7f52d7c6c640 -1 reset not still connected to 0x557844a984e0 2024-09-18T00:26:10.129 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:26:10.127+0000 7f52d7c6c640 -1 reset not still connected to 0x5578453d2270 2024-09-18T00:26:10.129 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:26:10.127+0000 7f52d7c6c640 -1 reset not still connected to 0x5578476c3380 2024-09-18T00:26:10.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1631 2024-09-18T00:26:10.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:26:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:26:11.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:26:11.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1632 2024-09-18T00:26:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:26:12.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 76084 2024-09-18T00:26:13.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1633 2024-09-18T00:26:13.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:26:13.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:26:13.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:26:13.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:26:13.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:26:13.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:26:13.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:14.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:26:14.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 76261 2024-09-18T00:26:15.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1634 2024-09-18T00:26:15.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:26:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:26:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 76558 2024-09-18T00:26:18.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:26:18.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:26:18.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 74929 2024-09-18T00:28:32.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:28:32.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:28:32.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:28:32.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 75063 2024-09-18T00:28:33.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:28:33.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:28:33.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:28:33.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 75201 2024-09-18T00:28:35.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:28:35.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:28:35.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:28:35.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 75373 2024-09-18T00:28:37.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:28:37.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:28:37.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:28:37.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 75521 2024-09-18T00:28:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:28:39.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:28:39.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:28:39.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 75695 2024-09-18T00:28:41.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:28:41.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:28:41.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:28:41.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 75894 2024-09-18T00:28:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:28:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:28:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:28:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 76085 2024-09-18T00:28:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:28:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:28:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:28:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 76262 2024-09-18T00:28:47.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:28:47.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:28:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:28:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 76559 2024-09-18T00:28:49.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:28:49.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:28:49.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:28:49.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:28:49.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:28:49.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:28:49.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 1 2024-09-18T00:28:49.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd0 2024-09-18T00:28:49.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:28:49.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary1 2024-09-18T00:28:49.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary1 2024-09-18T00:28:49.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:28:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:28:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary1 2024-09-18T00:28:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 2 2024-09-18T00:28:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd1 2024-09-18T00:28:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary2 2024-09-18T00:28:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary2 2024-09-18T00:28:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:28:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:28:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary2 2024-09-18T00:28:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:28:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 3 2024-09-18T00:28:49.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:28:49.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 4 2024-09-18T00:28:49.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd2 2024-09-18T00:28:49.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary3 2024-09-18T00:28:49.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:28:49.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary3 2024-09-18T00:28:49.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:28:49.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd3 2024-09-18T00:28:49.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary4 2024-09-18T00:28:49.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary4 2024-09-18T00:28:49.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary3 2024-09-18T00:28:49.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:28:49.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary4 2024-09-18T00:28:49.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:28:49.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:28:49.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 5 2024-09-18T00:28:49.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:28:49.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd4 2024-09-18T00:28:49.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary5 2024-09-18T00:28:49.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:28:49.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary5 2024-09-18T00:28:49.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 6 2024-09-18T00:28:49.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:28:49.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary5 2024-09-18T00:28:49.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd5 2024-09-18T00:28:49.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:28:49.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:28:49.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary6 2024-09-18T00:28:49.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary6 2024-09-18T00:28:49.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:28:49.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 7 2024-09-18T00:28:49.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary6 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd6 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary7 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary7 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary7 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 8 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd7 2024-09-18T00:28:49.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary8 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 82470 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary8 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 9 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary8 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd8 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary9 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary9 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 10 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary9 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd9 2024-09-18T00:28:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary10 2024-09-18T00:28:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary10 2024-09-18T00:28:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:28:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary10 2024-09-18T00:28:50.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:28:50.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:28:50.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:28:50.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:28:50.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:28:50.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:28:50.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:28:50.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:28:50.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:28:50.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:28:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:28:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:28:50.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:28:50.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:28:50.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:28:50.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:28:50.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:28:50.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:28:50.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:28:50.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:29:47.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=fca7aadf2e080616214711c37595848b 2024-09-18T00:29:47.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary8 2024-09-18T00:29:48.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=d78d83cc31c319b00cbe4f3fadfd7902 2024-09-18T00:29:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary5 2024-09-18T00:29:49.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary8 2024-09-18T00:29:49.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:49.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:49.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:29:49.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary8 2024-09-18T00:29:49.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=6ceffb862d284cf57f48ef8cd391c0d0 2024-09-18T00:29:49.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary7 2024-09-18T00:29:50.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary5 2024-09-18T00:29:50.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:50.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:50.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:29:50.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary5 2024-09-18T00:29:50.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=d07a1645c072627d43dbe8eafad2ea98 2024-09-18T00:29:50.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary4 2024-09-18T00:29:50.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:29:50.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+unknown 2024-09-18T00:29:50.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:50.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:50.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:29:50.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:50.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:50.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:50.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:50.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:50.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=af18f0418e87d41363a71d708570ee1c 2024-09-18T00:29:50.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary3 2024-09-18T00:29:50.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary7 2024-09-18T00:29:50.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:50.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:50.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:29:50.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary7 2024-09-18T00:29:50.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=0c3ecf13b12a8cf6f385f706610f3af0 2024-09-18T00:29:50.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary6 2024-09-18T00:29:51.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary4 2024-09-18T00:29:51.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:51.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:51.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:29:51.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary4 2024-09-18T00:29:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:29:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:51.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:51.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:29:51.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:51.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:51.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:51.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:51.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:51.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:29:51.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:29:51.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 51c15641-cc65-4aa0-a9cf-9e4d6425d92c 2024-09-18T00:29:51.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:29:51.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:29:51.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:51.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:49 2024-09-18T00:29:51.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:51.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:51.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:51.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:29:51.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:51' 2024-09-18T00:29:51.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:51.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:29:51.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:51.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:29:51.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary6 2024-09-18T00:29:51.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:51.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:51.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:29:51.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary6 2024-09-18T00:29:51.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary3 2024-09-18T00:29:51.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:51.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:51.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:29:51.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary3 2024-09-18T00:29:52.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:29:52.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+unknown 2024-09-18T00:29:52.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:52.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:52.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:29:52.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:52.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:52.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:52.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:52.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:52.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:29:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+unknown 2024-09-18T00:29:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:29:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:52.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:52.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:52.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:52.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=aa6124a3a31cde4f6702c3a6dd3f440f 2024-09-18T00:29:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary2 2024-09-18T00:29:53.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=fa3428528aba5be0553b509798869174 2024-09-18T00:29:53.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary1 2024-09-18T00:29:53.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:29:53.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:29:53.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:29:53.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:29:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:53.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:53.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:29:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+unknown 2024-09-18T00:29:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:29:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:53.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:53.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:29:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+unknown 2024-09-18T00:29:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:53.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:29:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:29:53.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:53.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:53.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:29:53.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:53.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:53.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:53.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:53.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:53.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:29:53.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+unknown 2024-09-18T00:29:53.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:53.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:53.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:29:53.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:53.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:53.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:53.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:53.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:53.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:29:53.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4fd082f9-a5cb-42cc-9a45-3ebe1a50bbca 2024-09-18T00:29:53.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:49 2024-09-18T00:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:53' 2024-09-18T00:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:53.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:29:53.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:53.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:29:53.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:29:53.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 068a799c-59b8-4a9a-acb8-c91e7357896b 2024-09-18T00:29:53.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:29:53.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:29:53.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:53.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:49 2024-09-18T00:29:53.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:53.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:53.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:53.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:29:53.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:53' 2024-09-18T00:29:53.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:53.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:29:53.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:53.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:29:53.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:29:53.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:53.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:53.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:29:53.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:53.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:53.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:53.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:53.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:53.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:29:53.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary2 2024-09-18T00:29:53.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:53.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:53.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:29:53.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary2 2024-09-18T00:29:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:29:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 51c15641-cc65-4aa0-a9cf-9e4d6425d92c 2024-09-18T00:29:53.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:29:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:29:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:49 2024-09-18T00:29:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:29:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:51' 2024-09-18T00:29:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:29:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:29:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary1 2024-09-18T00:29:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:29:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary1 2024-09-18T00:29:54.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=4aa6e2f750a01d333767e60512eb6400 2024-09-18T00:29:54.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary9 2024-09-18T00:29:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:29:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:54.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:29:54.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:54.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:54.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:54.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:54.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:54.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:29:54.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T00:29:54.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:54.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:54.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:29:54.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:54.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:54.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:54.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:54.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:54.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:29:54.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T00:29:54.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:54.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:54.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:29:54.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:54.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:54.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:54.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:54.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:54.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:29:54.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:29:54.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f28d710c-fce9-48ab-b72f-a7c632f6c1ff 2024-09-18T00:29:54.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:54.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:29:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2164736.0,"last_snapshot_bytes":2555904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619162,"remote_snapshot_timestamp":1726619162,"replay_state":"idle"} 2024-09-18T00:29:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:39' 2024-09-18T00:29:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:54.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:54.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:54.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:29:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:29:54.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:54.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:29:54.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:29:54.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:29:54.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:29:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:29:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: af284746-4562-4e55-b3a1-340279f4d151 2024-09-18T00:29:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:54.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:54.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:54.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:54.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:29:54.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3203072.0,"last_snapshot_bytes":4710400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619161,"remote_snapshot_timestamp":1726619161,"replay_state":"idle"} 2024-09-18T00:29:54.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:39' 2024-09-18T00:29:54.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:54.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:54.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:54.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:29:54.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:29:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:29:54.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:29:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:29:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:29:54.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:54.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+unknown '2024-09-18 00:29:54' 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+unknown '2024-09-18 00:29:54' 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:54' 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:54' 2024-09-18T00:29:54.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:29:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:54.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:29:54.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:29:54.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: b713c9d1-2c92-4cf8-9e0a-982f46d2d359 2024-09-18T00:29:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:29:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2162688.0,"last_snapshot_bytes":2535424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619165,"remote_snapshot_timestamp":1726619165,"replay_state":"idle"} 2024-09-18T00:29:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:39' 2024-09-18T00:29:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:54.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:29:54.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:54.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:54.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:29:54.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:54.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:29:54.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:29:54.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:29:54.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:29:54.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:54.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+unknown '2024-09-18 00:29:54' 2024-09-18T00:29:54.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:54.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:54.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:29:54.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:54.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:54' 2024-09-18T00:29:54.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:54.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:54.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:29:54.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1699840.0,"last_snapshot_bytes":1724416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619156,"remote_snapshot_timestamp":1726619156,"replay_state":"idle"}2024-09-18 00:29:39image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2621440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619174,"remote_snapshot_timestamp":1726619174,"replay_state":"idle"}2024-09-18 00:29:39image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3205120.0,"last_snapshot_bytes":2605056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619158,"remote_snapshot_timestamp":1726619158,"replay_state":"idle"}2024-09-18 00:29:39image-primary3af284746-4562-4e55-b3a1-340279f4d151up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3203072.0,"last_snapshot_bytes":4710400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619161,"remote_snapshot_timestamp":1726619161,"replay_state":"idle"}2024-09-18 00:29:39image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2164736.0,"last_snapshot_bytes":2555904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619162,"remote_snapshot_timestamp":1726619162,"replay_state":"idle"}2024-09-18 00:29:39image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2162688.0,"last_snapshot_bytes":2535424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619165,"remote_snapshot_timestamp":1726619165,"replay_state":"idle"}2024-09-18 00:29:39image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2639872.0,"last_snapshot_bytes":229376,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619172,"remote_snapshot_timestamp":1726619172,"replay_state":"idle"}2024-09-18 00:29:39' 2024-09-18T00:29:54.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:29:54.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:29:54.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1699840.0,"last_snapshot_bytes":1724416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619156,"remote_snapshot_timestamp":1726619156,"replay_state":"idle"}2024-09-18 00:29:39image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2621440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619174,"remote_snapshot_timestamp":1726619174,"replay_state":"idle"}2024-09-18 00:29:39image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3205120.0,"last_snapshot_bytes":2605056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619158,"remote_snapshot_timestamp":1726619158,"replay_state":"idle"}2024-09-18 00:29:39image-primary3af284746-4562-4e55-b3a1-340279f4d151up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3203072.0,"last_snapshot_bytes":4710400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619161,"remote_snapshot_timestamp":1726619161,"replay_state":"idle"}2024-09-18 00:29:39image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2164736.0,"last_snapshot_bytes":2555904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619162,"remote_snapshot_timestamp":1726619162,"replay_state":"idle"}2024-09-18 00:29:39image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2162688.0,"last_snapshot_bytes":2535424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619165,"remote_snapshot_timestamp":1726619165,"replay_state":"idle"}2024-09-18 00:29:39image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2639872.0,"last_snapshot_bytes":229376,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619172,"remote_snapshot_timestamp":1726619172,"replay_state":"idle"}2024-09-18 00:29:39' 2024-09-18T00:29:54.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:29:54.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:29:54.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1699840.0,"last_snapshot_bytes":1724416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619156,"remote_snapshot_timestamp":1726619156,"replay_state":"idle"}2024-09-18 00:29:39image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2621440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619174,"remote_snapshot_timestamp":1726619174,"replay_state":"idle"}2024-09-18 00:29:39image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3205120.0,"last_snapshot_bytes":2605056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619158,"remote_snapshot_timestamp":1726619158,"replay_state":"idle"}2024-09-18 00:29:39image-primary3af284746-4562-4e55-b3a1-340279f4d151up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3203072.0,"last_snapshot_bytes":4710400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619161,"remote_snapshot_timestamp":1726619161,"replay_state":"idle"}2024-09-18 00:29:39image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2164736.0,"last_snapshot_bytes":2555904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619162,"remote_snapshot_timestamp":1726619162,"replay_state":"idle"}2024-09-18 00:29:39image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2162688.0,"last_snapshot_bytes":2535424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619165,"remote_snapshot_timestamp":1726619165,"replay_state":"idle"}2024-09-18 00:29:39image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2639872.0,"last_snapshot_bytes":229376,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619172,"remote_snapshot_timestamp":1726619172,"replay_state":"idle"}2024-09-18 00:29:39' 2024-09-18T00:29:54.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:29:54.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:29:54.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary9 2024-09-18T00:29:54.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:54.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:54.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:29:54.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary9 2024-09-18T00:29:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:29:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:29:54.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:29:54.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:29:54.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:29:54.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:54.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:54.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:29:54.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:29:54.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:29:54.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:54.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:29:54.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:54.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary6 up+unknown 2024-09-18T00:29:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:29:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary4 up+unknown 2024-09-18T00:29:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:29:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:54.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:54.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:54.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:54.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary3 up+unknown 2024-09-18T00:29:54.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:54.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:54.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:29:54.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:54.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:54.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:54.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:54.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:55.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:29:55.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:29:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+unknown 2024-09-18T00:29:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:29:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:55.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:55.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:55.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+unknown 2024-09-18T00:29:55.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:55.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:55.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:29:55.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:55.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:55.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:55.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:55.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:55.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:29:55.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:55.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:55.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:29:55.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:55.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:55.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:55.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:55.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:55.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:29:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:29:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:29:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:55.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:29:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:29:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4fd082f9-a5cb-42cc-9a45-3ebe1a50bbca 2024-09-18T00:29:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54' 2024-09-18T00:29:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:55.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:29:55.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:55.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:55.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:29:55.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:55.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:29:55.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:29:55.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:29:55.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:29:55.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:55.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+unknown '2024-09-18 00:29:54' 2024-09-18T00:29:55.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:55.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:55.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:29:55.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:55.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:54' 2024-09-18T00:29:55.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:55.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:55.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:29:55.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:29:55.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 068a799c-59b8-4a9a-acb8-c91e7357896b 2024-09-18T00:29:55.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:55.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:55.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:55.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:55.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:55.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:55.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:55.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:55.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54' 2024-09-18T00:29:55.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:55.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:55.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:55.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:29:55.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:55.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:55.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:29:55.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:55.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:29:55.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:29:55.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:29:55.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:29:55.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:55.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+unknown '2024-09-18 00:29:54' 2024-09-18T00:29:55.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:55.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:55.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:29:55.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:55.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:54' 2024-09-18T00:29:55.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:55.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:55.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:29:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=f53a7f2545a6fa0b4dc76ab2965cb284 2024-09-18T00:29:55.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary10 2024-09-18T00:29:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary4 up+unknown '' '' 2024-09-18T00:29:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:29:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary3 up+unknown '' '' 2024-09-18T00:29:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary6 up+unknown '' '' 2024-09-18T00:29:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary4 2024-09-18T00:29:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary3 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary6 2024-09-18T00:29:56.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1699840.0,"last_snapshot_bytes":1724416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619156,"remote_snapshot_timestamp":1726619156,"replay_state":"idle"}2024-09-18 00:29:39image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2621440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619174,"remote_snapshot_timestamp":1726619174,"replay_state":"idle"}2024-09-18 00:29:39image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3205120.0,"last_snapshot_bytes":2605056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619158,"remote_snapshot_timestamp":1726619158,"replay_state":"idle"}2024-09-18 00:29:39image-primary3af284746-4562-4e55-b3a1-340279f4d151up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:55image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:55image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:55image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2639872.0,"last_snapshot_bytes":229376,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619172,"remote_snapshot_timestamp":1726619172,"replay_state":"idle"}2024-09-18 00:29:39' 2024-09-18T00:29:56.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:29:56.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:29:56.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:56.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:29:56.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1699840.0,"last_snapshot_bytes":1724416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619156,"remote_snapshot_timestamp":1726619156,"replay_state":"idle"}2024-09-18 00:29:39image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2621440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619174,"remote_snapshot_timestamp":1726619174,"replay_state":"idle"}2024-09-18 00:29:39image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3205120.0,"last_snapshot_bytes":2605056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619158,"remote_snapshot_timestamp":1726619158,"replay_state":"idle"}2024-09-18 00:29:39image-primary3af284746-4562-4e55-b3a1-340279f4d151up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:55image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:55image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:55image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2639872.0,"last_snapshot_bytes":229376,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619172,"remote_snapshot_timestamp":1726619172,"replay_state":"idle"}2024-09-18 00:29:39' 2024-09-18T00:29:56.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:29:56.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:29:56.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:29:56.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:56.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:29:56.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary5 up+unknown 2024-09-18T00:29:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:29:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:56.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:56.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:56.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:29:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:29:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:29:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary7 up+unknown 2024-09-18T00:29:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:29:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:56.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:29:56.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: af284746-4562-4e55-b3a1-340279f4d151 2024-09-18T00:29:56.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:56.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:29:56.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:56.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:55 2024-09-18T00:29:56.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:56.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:29:56.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:56.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:56.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54' 2024-09-18T00:29:56.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:56.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:56.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:56.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:29:56.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:56.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:56.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:29:56.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:56.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:29:56.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:29:56.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:29:56.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:29:56.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:55' 2024-09-18T00:29:56.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary3 up+unknown '2024-09-18 00:29:55' 2024-09-18T00:29:56.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:56.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:56.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:29:56.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:56.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:55' 2024-09-18T00:29:56.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:56.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:56.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:29:56.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:29:56.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f28d710c-fce9-48ab-b72f-a7c632f6c1ff 2024-09-18T00:29:56.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:56.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:29:56.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:56.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:55 2024-09-18T00:29:56.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:56.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:29:56.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:56.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:56.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54' 2024-09-18T00:29:56.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:56.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:56.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:29:56.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: b713c9d1-2c92-4cf8-9e0a-982f46d2d359 2024-09-18T00:29:56.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:56.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:29:56.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:56.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:55 2024-09-18T00:29:56.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:56.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:29:56.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:56.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:56.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54' 2024-09-18T00:29:56.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:56.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:56.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:56.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:29:56.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:56.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:56.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:56.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:29:56.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:29:56.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:56.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:56.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:56.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:29:56.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:29:56.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:29:56.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:56.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:29:56.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:29:56.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:29:56.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:29:56.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:29:56.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:29:56.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:55' 2024-09-18T00:29:56.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary4 up+unknown '2024-09-18 00:29:55' 2024-09-18T00:29:56.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:56.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:56.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:29:56.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:56.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:55' 2024-09-18T00:29:56.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:29:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:55' 2024-09-18T00:29:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary6 up+unknown '2024-09-18 00:29:55' 2024-09-18T00:29:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:29:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:55' 2024-09-18T00:29:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:56.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:29:56.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1699840.0,"last_snapshot_bytes":1724416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619156,"remote_snapshot_timestamp":1726619156,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:29:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2621440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619174,"remote_snapshot_timestamp":1726619174,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:29:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3205120.0,"last_snapshot_bytes":2605056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619158,"remote_snapshot_timestamp":1726619158,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:29:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary3af284746-4562-4e55-b3a1-340279f4d151up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2639872.0,"last_snapshot_bytes":229376,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619172,"remote_snapshot_timestamp":1726619172,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:29:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54' 2024-09-18T00:29:56.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:29:56.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:29:56.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:55' 2024-09-18T00:29:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:29:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1699840.0,"last_snapshot_bytes":1724416,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619156,"remote_snapshot_timestamp":1726619156,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:29:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2621440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619174,"remote_snapshot_timestamp":1726619174,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:29:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":3205120.0,"last_snapshot_bytes":2605056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619158,"remote_snapshot_timestamp":1726619158,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:29:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary3af284746-4562-4e55-b3a1-340279f4d151up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2639872.0,"last_snapshot_bytes":229376,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619172,"remote_snapshot_timestamp":1726619172,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:29:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54' 2024-09-18T00:29:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:29:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:29:56.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:55' 2024-09-18T00:29:56.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:29:56.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:29:56.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:56.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:29:56.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:56.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:56.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:56.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary4 2024-09-18T00:29:56.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:56.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:56.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:29:56.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:29:56.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary4 2024-09-18T00:29:56.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:29:56.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:56.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:29:56.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary3 2024-09-18T00:29:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:29:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:29:56.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary3 2024-09-18T00:29:56.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+stopping_replayremote image demoted42614264mirror.0smithi0492024-09-18 00:29:56cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2621440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619174,"remote_snapshot_timestamp":1726619174,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:29:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+stopping_replayremote image demoted42614264mirror.0smithi0492024-09-18 00:29:56cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary3af284746-4562-4e55-b3a1-340279f4d151up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2639872.0,"last_snapshot_bytes":229376,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619172,"remote_snapshot_timestamp":1726619172,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:29:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54' 2024-09-18T00:29:56.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:29:56.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:29:56.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:55' 2024-09-18T00:29:56.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:29:56.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:29:56.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:56.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:29:56.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:56.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:56.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:56.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary6 2024-09-18T00:29:56.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:56.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:56.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:29:56.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:29:56.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary6 2024-09-18T00:29:56.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:29:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+unknown 2024-09-18T00:29:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:29:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:56.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:56.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:56.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:56.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:29:56.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:56.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:29:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:56.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:29:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:29:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:29:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:56.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:29:56.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary10 2024-09-18T00:29:56.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:56.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:56.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:29:56.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary10 2024-09-18T00:29:56.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:29:56.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 838864e7-c9c1-4cb7-9af3-e63a599ff8a4 2024-09-18T00:29:56.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:29:56.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:29:56.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:56.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:56.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:56.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:56.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:56.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:29:56.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:56' 2024-09-18T00:29:56.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:56.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:29:56.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:56.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:29:56.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:29:56.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: d18e1596-14e2-4da8-9e9a-daeec5ecfa30 2024-09-18T00:29:56.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:29:56.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:29:56.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:56.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:56.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:56.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:56.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:56.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:29:56.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:56' 2024-09-18T00:29:56.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:56.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:29:56.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:56.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:29:57.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary5 up+unknown '' '' 2024-09-18T00:29:57.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:57.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:57.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:29:57.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:57.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:57.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:57.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:57.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:57.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary5 2024-09-18T00:29:57.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary7 up+unknown '' '' 2024-09-18T00:29:57.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:57.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:57.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:29:57.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:57.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:57.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:57.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:57.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:57.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary7 2024-09-18T00:29:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:29:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4fd082f9-a5cb-42cc-9a45-3ebe1a50bbca 2024-09-18T00:29:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:29:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54' 2024-09-18T00:29:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:29:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:57.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:57.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:29:57.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:57.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:29:57.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:29:57.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:29:57.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:29:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:29:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 068a799c-59b8-4a9a-acb8-c91e7357896b 2024-09-18T00:29:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:57.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:57.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:29:57.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:57.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:57.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54' 2024-09-18T00:29:57.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:57.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:57.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary5 up+unknown '2024-09-18 00:29:54' 2024-09-18T00:29:57.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:57.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:57.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:29:57.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:57.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:54' 2024-09-18T00:29:57.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:57.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:57.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:57.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:57.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:29:57.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:29:57.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:57.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:57.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:29:57.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:57.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:29:57.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:29:57.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:29:57.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:29:57.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:57.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary7 up+unknown '2024-09-18 00:29:54' 2024-09-18T00:29:57.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:57.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:57.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:29:57.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:57.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:54' 2024-09-18T00:29:57.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:57.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:57.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:29:57.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:29:57.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:29:57.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:29:57.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:29:57.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:56cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2621440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619174,"remote_snapshot_timestamp":1726619174,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:29:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:56cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary3af284746-4562-4e55-b3a1-340279f4d151up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+stopping_replayremote image demoted42614264mirror.0smithi0492024-09-18 00:29:57cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54' 2024-09-18T00:29:57.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:29:57.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:29:57.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:29:57.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:57.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:29:57.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:29:57.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:56cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2621440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619174,"remote_snapshot_timestamp":1726619174,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:29:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:56cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54image-primary3af284746-4562-4e55-b3a1-340279f4d151up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:55cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+stopping_replayremote image demoted42614264mirror.0smithi0492024-09-18 00:29:57cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:29:54' 2024-09-18T00:29:57.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:29:57.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:29:57.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:29:57.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:29:57.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:57.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:57.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:57.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:57.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary5 2024-09-18T00:29:57.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:57.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:57.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:29:57.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:29:57.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary5 2024-09-18T00:29:57.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:57.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:29:57.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:29:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:29:57.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:57.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:57.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:57.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary7 2024-09-18T00:29:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:29:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:29:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary7 2024-09-18T00:29:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:29:57.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:29:57.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd0 2024-09-18T00:29:57.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:29:57.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:29:57.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd1 2024-09-18T00:29:57.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:29:57.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:29:57.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd2 2024-09-18T00:29:57.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:29:57.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:29:57.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T00:29:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:29:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:57.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:29:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:29:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a4a49402-ee8f-4527-b9d9-e3c5d7289204 2024-09-18T00:29:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:29:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:29:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:57.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:57.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:57.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:57.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:57.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:29:57.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:57' 2024-09-18T00:29:57.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:57.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:29:57.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:57.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:29:57.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:29:57.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:57.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:57.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:29:57.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:57.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:57.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:57.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:57.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:57.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:29:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:29:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 51c15641-cc65-4aa0-a9cf-9e4d6425d92c 2024-09-18T00:29:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54' 2024-09-18T00:29:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:58.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:58.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:29:58.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:58.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:58.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:29:58.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:58.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:29:58.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:29:58.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:29:58.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:29:58.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:58.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+unknown '2024-09-18 00:29:54' 2024-09-18T00:29:58.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:58.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:58.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:29:58.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:58.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:54' 2024-09-18T00:29:58.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:58.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:58.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:29:58.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:29:58.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:29:58.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:29:58.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:29:58.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:29:58.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:29:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:56image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2621440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619174,"remote_snapshot_timestamp":1726619174,"replay_state":"idle"}2024-09-18 00:29:39image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:56image-primary3af284746-4562-4e55-b3a1-340279f4d151up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:55image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:55image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:55image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:29:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:57' 2024-09-18T00:29:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:29:58.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:29:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:54' 2024-09-18T00:29:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:29:58.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:29:58.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:58.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:29:58.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:58.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:58.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:29:58.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary8 up+unknown 2024-09-18T00:29:58.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:58.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:58.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:29:58.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:58.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:58.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:58.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:58.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:58.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd3 2024-09-18T00:29:58.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:29:58.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:29:58.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd4 2024-09-18T00:29:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:29:58.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:29:58.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:29:58.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+unknown 2024-09-18T00:29:58.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:58.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:58.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:29:58.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:58.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:58.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:58.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:58.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:29:58.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:29:58.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:58.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:58.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:29:58.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:58.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:58.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:58.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:58.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:58.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:29:58.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:29:58.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:58.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:58.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:29:58.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:58.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:58.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:58.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:58.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:58.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:29:58.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:29:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: d18e1596-14e2-4da8-9e9a-daeec5ecfa30 2024-09-18T00:29:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:29:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:29:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:29:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:56' 2024-09-18T00:29:58.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:58.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:29:58.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:58.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:29:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:29:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 838864e7-c9c1-4cb7-9af3-e63a599ff8a4 2024-09-18T00:29:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:29:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:29:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:54 2024-09-18T00:29:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:29:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:56' 2024-09-18T00:29:58.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:29:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:29:58.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:29:59.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary8 up+unknown '' '' 2024-09-18T00:29:59.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:59.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:59.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:29:59.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:59.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:59.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:59.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:59.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:59.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary8 2024-09-18T00:29:59.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T00:29:59.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:59.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:59.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:29:59.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:59.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:59.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:59.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:59.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:59.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:29:59.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:29:59.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 51c15641-cc65-4aa0-a9cf-9e4d6425d92c 2024-09-18T00:29:59.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:59.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:59.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:59.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59 2024-09-18T00:29:59.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:59.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:29:59.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:59.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:59.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59' 2024-09-18T00:29:59.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:59.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:59.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:59.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:29:59.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:59.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:59.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:29:59.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:29:59.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:29:59.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:29:59.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:29:59.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:29:59.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T00:29:59.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:29:59.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:59.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:29:59.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:59.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:29:59.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:29:59.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:29:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary8 up+unknown '2024-09-18 00:29:59' 2024-09-18T00:29:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:29:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:29:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:29:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:29:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:59' 2024-09-18T00:29:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:29:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:59.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:29:59.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:29:59.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:29:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:29:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a4a49402-ee8f-4527-b9d9-e3c5d7289204 2024-09-18T00:29:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:29:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59 2024-09-18T00:29:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:29:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:29:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:29:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:29:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59' 2024-09-18T00:29:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:29:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:29:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:29:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:29:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:29:59.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:29:59.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:30:00.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:00.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:30:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:30:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:30:00.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:30:00.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:00.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+unknown '2024-09-18 00:29:59' 2024-09-18T00:30:00.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:30:00.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:00.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:30:00.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:00.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:59' 2024-09-18T00:30:00.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:00.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:00.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:30:00.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59image-primary3af284746-4562-4e55-b3a1-340279f4d151up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}2024-09-18 00:29:59image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}2024-09-18 00:29:59image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59' 2024-09-18T00:30:00.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:30:00.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:30:00.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:00.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:30:00.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:30:00.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:30:00.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:00.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:00.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:00.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary8 2024-09-18T00:30:00.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:00.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:00.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:30:00.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:30:00.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary8 2024-09-18T00:30:00.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:30:00.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4ce86fed-82d5-428f-affa-37bf7b004ee8 2024-09-18T00:30:00.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:00.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:30:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:30:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59 2024-09-18T00:30:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:30:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:30:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:30:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59' 2024-09-18T00:30:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:30:00.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:30:00.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:30:00.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:30:00.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:30:00.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:00.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:30:00.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:00.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:30:00.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:30:00.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:30:00.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:30:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+unknown '2024-09-18 00:29:59' 2024-09-18T00:30:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:30:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:30:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:59' 2024-09-18T00:30:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:30:00.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:59image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59image-primary3af284746-4562-4e55-b3a1-340279f4d151up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59' 2024-09-18T00:30:00.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:30:00.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:30:00.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:00.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:30:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:30:00.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:00.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:30:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:00.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary9 up+unknown 2024-09-18T00:30:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:30:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:30:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:30:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:30:00.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:30:00.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:59image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59image-primary3af284746-4562-4e55-b3a1-340279f4d151up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59' 2024-09-18T00:30:00.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:30:00.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:30:00.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:00.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:30:00.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:30:00.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:00.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:30:00.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:00.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:00.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:00.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary10 up+unknown 2024-09-18T00:30:00.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:00.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:00.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:30:00.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:00.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:30:00.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:30:00.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:30:00.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:30:01.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:30:01.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:30:01.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:30:01.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd5 2024-09-18T00:30:01.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:30:01.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:30:01.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary9 up+unknown '' '' 2024-09-18T00:30:01.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:30:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:30:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:30:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:01.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:01.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary9 2024-09-18T00:30:01.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary10 up+unknown '' '' 2024-09-18T00:30:01.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:01.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:01.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:30:01.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:01.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:30:01.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:30:01.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:01.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:01.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary10 2024-09-18T00:30:01.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:30:01.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a4a49402-ee8f-4527-b9d9-e3c5d7289204 2024-09-18T00:30:01.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:01.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:30:01.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:30:01.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59 2024-09-18T00:30:01.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:30:01.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:30:01.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:01.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:30:01.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59' 2024-09-18T00:30:01.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:30:01.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:30:01.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:30:01.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:30:01.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:30:01.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:01.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:30:01.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:30:01.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4ce86fed-82d5-428f-affa-37bf7b004ee8 2024-09-18T00:30:01.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:01.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:30:01.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:30:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59 2024-09-18T00:30:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:30:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:30:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:30:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59' 2024-09-18T00:30:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:30:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:01.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:30:01.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:30:01.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:30:01.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:30:01.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:30:01.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:30:01.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:01.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:30:01.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:30:01.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:30:01.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:01.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:30:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:30:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:30:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:30:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary9 up+unknown '2024-09-18 00:29:59' 2024-09-18T00:30:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:01.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:30:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:59' 2024-09-18T00:30:01.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:01.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:01.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary10 up+unknown '2024-09-18 00:29:59' 2024-09-18T00:30:01.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:01.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:01.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:30:01.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:01.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:59' 2024-09-18T00:30:01.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:01.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:01.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:30:01.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:01.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:30:02.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59image-primary3af284746-4562-4e55-b3a1-340279f4d151up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}2024-09-18 00:29:59image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}2024-09-18 00:29:59image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59' 2024-09-18T00:30:02.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:30:02.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:30:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:30:02.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:30:02.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:02.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:30:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary9 2024-09-18T00:30:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:02.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:02.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:30:02.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:30:02.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary9 2024-09-18T00:30:02.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59image-primary3af284746-4562-4e55-b3a1-340279f4d151up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}2024-09-18 00:29:59image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}2024-09-18 00:29:59image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:29:59cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:29:59' 2024-09-18T00:30:02.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:30:02.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:30:02.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:02.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:30:02.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:30:02.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:02.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:30:02.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:02.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:02.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:02.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary10 2024-09-18T00:30:02.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:02.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:02.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:30:02.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:30:02.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary10 2024-09-18T00:30:02.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:30:02.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:30:02.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:02.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:30:02.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:02.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:30:02.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:30:02.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:02.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:02.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:30:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:30:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:30:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:02.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:30:02.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:30:02.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: d18e1596-14e2-4da8-9e9a-daeec5ecfa30 2024-09-18T00:30:02.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:03.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:30:03.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:30:03.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59 2024-09-18T00:30:03.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:30:03.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:30:03.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:03.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:30:03.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59' 2024-09-18T00:30:03.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:30:03.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:30:03.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:30:03.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:30:03.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:30:03.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:30:03.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:03.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:03.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:30:03.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:30:03.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:30:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:30:03.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:30:03.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 838864e7-c9c1-4cb7-9af3-e63a599ff8a4 2024-09-18T00:30:03.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:03.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:30:03.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:30:03.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59 2024-09-18T00:30:03.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:30:03.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:30:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:30:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:29:59' 2024-09-18T00:30:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:30:03.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:30:03.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:30:03.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:30:03.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:30:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:30:03.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:03.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:03.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:30:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:30:03.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:30:03.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:30:03.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:03.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+unknown '2024-09-18 00:29:59' 2024-09-18T00:30:03.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:30:03.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:03.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:30:03.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:03.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:59' 2024-09-18T00:30:03.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:03.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:03.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:30:03.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:03.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+unknown '2024-09-18 00:29:59' 2024-09-18T00:30:03.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:30:03.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:03.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:30:03.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:03.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:29:59' 2024-09-18T00:30:03.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:03.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:03.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:30:03.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:59image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59image-primary3af284746-4562-4e55-b3a1-340279f4d151up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59' 2024-09-18T00:30:03.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:30:03.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:30:03.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:03.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:30:03.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:30:03.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:30:03.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:03.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:30:03.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:03.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:03.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:03.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary1 up+unknown 2024-09-18T00:30:03.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:03.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:03.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:30:03.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:03.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:30:03.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:30:03.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:30:03.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:30:03.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:30:03.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:29:59image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59image-primary3af284746-4562-4e55-b3a1-340279f4d151up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:29:59image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:29:59cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:29:59' 2024-09-18T00:30:03.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:30:03.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:30:03.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:29:59' 2024-09-18T00:30:03.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:30:03.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:30:03.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:03.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:30:03.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:03.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:03.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary2 up+unknown 2024-09-18T00:30:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:30:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:30:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:30:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:30:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:30:03.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:30:03.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:30:03.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:30:03.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:30:03.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd6 2024-09-18T00:30:03.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:30:03.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:30:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd7 2024-09-18T00:30:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:30:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:30:04.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary1 up+unknown '' '' 2024-09-18T00:30:04.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:04.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:04.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:30:04.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:04.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:30:04.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:30:04.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:04.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:04.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary1 2024-09-18T00:30:04.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary2 up+unknown '' '' 2024-09-18T00:30:04.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:04.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:04.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:30:04.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:04.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:30:04.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:30:04.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:04.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary2 2024-09-18T00:30:04.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:30:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: d18e1596-14e2-4da8-9e9a-daeec5ecfa30 2024-09-18T00:30:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:30:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:30:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:30:04 2024-09-18T00:30:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:30:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:30:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:04.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:30:04.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:30:04' 2024-09-18T00:30:04.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:30:04.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:30:04.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:30:04.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:30:04.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:30:04.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:04.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:30:04.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:04.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:30:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:30:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:30:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:30:04.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:30:04' 2024-09-18T00:30:04.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary1 up+unknown '2024-09-18 00:30:04' 2024-09-18T00:30:04.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:04.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:04.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:30:04.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:04.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:30:04' 2024-09-18T00:30:04.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:04.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:04.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:30:04.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:30:04.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 838864e7-c9c1-4cb7-9af3-e63a599ff8a4 2024-09-18T00:30:04.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:04.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:30:04.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:30:04.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:30:04 2024-09-18T00:30:04.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:30:04.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:30:04.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:30:04.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:30:04.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:30:04' 2024-09-18T00:30:04.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:30:04.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:30:04.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:30:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:30:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:30:04.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:30:04.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:04.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:04.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:30:04.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:30:04.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:30:04.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:30:04.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:30:04' 2024-09-18T00:30:04.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary2 up+unknown '2024-09-18 00:30:04' 2024-09-18T00:30:04.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:04.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:04.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:30:04.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:30:04.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:30:04' 2024-09-18T00:30:04.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:30:04.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:30:04.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:30:04.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING2842614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:30:04image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}2024-09-18 00:30:04image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:30:04image-primary3af284746-4562-4e55-b3a1-340279f4d151up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}2024-09-18 00:29:59image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}2024-09-18 00:29:59image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619401,"replay_state":"idle"}2024-09-18 00:30:04image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}2024-09-18 00:30:04' 2024-09-18T00:30:04.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:30:04.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:30:04.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:30:04' 2024-09-18T00:30:04.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:30:04.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:30:04.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:30:04.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:04.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:04.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:04.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:04.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary1 2024-09-18T00:30:04.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:04.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:04.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:30:04.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:30:04.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary1 2024-09-18T00:30:05.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING2842614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:30:04image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}2024-09-18 00:30:04image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:30:04image-primary3af284746-4562-4e55-b3a1-340279f4d151up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}2024-09-18 00:29:59image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}2024-09-18 00:29:59image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}2024-09-18 00:29:59image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619401,"replay_state":"idle"}2024-09-18 00:30:04image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:30:04cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}2024-09-18 00:30:04' 2024-09-18T00:30:05.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:30:05.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:30:05.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:30:04' 2024-09-18T00:30:05.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:30:05.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:30:05.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:30:05.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:30:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:30:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:30:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary2 2024-09-18T00:30:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:30:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:30:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:30:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:30:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary2 2024-09-18T00:30:06.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:30:06.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:30:06.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:30:06.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:30:06.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:30:06.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:30:06.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd8 2024-09-18T00:30:06.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:30:06.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:30:07.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd9 2024-09-18T00:30:07.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:30:07.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:31:00.234 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:31:00.231+0000 7f52d7c6c640 -1 reset not still connected to 0x55785ab60a90 2024-09-18T00:31:04.206 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:31:04.204+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735aedd0 2024-09-18T00:31:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=d07a1645c072627d43dbe8eafad2ea98 2024-09-18T00:31:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd0 2024-09-18T00:31:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=0c3ecf13b12a8cf6f385f706610f3af0 2024-09-18T00:31:44.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd1 2024-09-18T00:31:46.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=d78d83cc31c319b00cbe4f3fadfd7902 2024-09-18T00:31:46.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd4 2024-09-18T00:31:47.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ d07a1645c072627d43dbe8eafad2ea98 != \d\0\7\a\1\6\4\5\c\0\7\2\6\2\7\d\4\3\d\b\e\8\e\a\f\a\d\2\e\a\9\8 ]] 2024-09-18T00:31:47.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=af18f0418e87d41363a71d708570ee1c 2024-09-18T00:31:47.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd2 2024-09-18T00:31:47.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 0c3ecf13b12a8cf6f385f706610f3af0 != \0\c\3\e\c\f\1\3\b\1\2\a\8\c\f\6\f\3\8\5\f\7\0\6\6\1\0\f\3\a\f\0 ]] 2024-09-18T00:31:47.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ d78d83cc31c319b00cbe4f3fadfd7902 != \d\7\8\d\8\3\c\c\3\1\c\3\1\9\b\0\0\c\b\e\4\f\3\f\a\d\f\d\7\9\0\2 ]] 2024-09-18T00:31:48.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=6ceffb862d284cf57f48ef8cd391c0d0 2024-09-18T00:31:48.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd3 2024-09-18T00:31:48.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ af18f0418e87d41363a71d708570ee1c != \a\f\1\8\f\0\4\1\8\e\8\7\d\4\1\3\6\3\a\7\1\d\7\0\8\5\7\0\e\e\1\c ]] 2024-09-18T00:31:48.859 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:31:48.846+0000 7f52d7c6c640 -1 reset not still connected to 0x557874e4f040 2024-09-18T00:31:48.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 6ceffb862d284cf57f48ef8cd391c0d0 != \6\c\e\f\f\b\8\6\2\d\2\8\4\c\f\5\7\f\4\8\e\f\8\c\d\3\9\1\c\0\d\0 ]] 2024-09-18T00:31:50.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=f53a7f2545a6fa0b4dc76ab2965cb284 2024-09-18T00:31:50.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd7 2024-09-18T00:31:50.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=fca7aadf2e080616214711c37595848b 2024-09-18T00:31:50.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd5 2024-09-18T00:31:51.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ f53a7f2545a6fa0b4dc76ab2965cb284 != \f\5\3\a\7\f\2\5\4\5\a\6\f\a\0\b\4\d\c\7\6\a\b\2\9\6\5\c\b\2\8\4 ]] 2024-09-18T00:31:51.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ fca7aadf2e080616214711c37595848b != \f\c\a\7\a\a\d\f\2\e\0\8\0\6\1\6\2\1\4\7\1\1\c\3\7\5\9\5\8\4\8\b ]] 2024-09-18T00:31:51.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=4aa6e2f750a01d333767e60512eb6400 2024-09-18T00:31:51.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd6 2024-09-18T00:31:51.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=aa6124a3a31cde4f6702c3a6dd3f440f 2024-09-18T00:31:51.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd8 2024-09-18T00:31:52.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 4aa6e2f750a01d333767e60512eb6400 != \4\a\a\6\e\2\f\7\5\0\a\0\1\d\3\3\3\7\6\7\e\6\0\5\1\2\e\b\6\4\0\0 ]] 2024-09-18T00:31:52.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ aa6124a3a31cde4f6702c3a6dd3f440f != \a\a\6\1\2\4\a\3\a\3\1\c\d\e\4\f\6\7\0\2\c\3\a\6\d\d\3\f\4\4\0\f ]] 2024-09-18T00:31:53.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=fa3428528aba5be0553b509798869174 2024-09-18T00:31:53.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd9 2024-09-18T00:31:53.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ fa3428528aba5be0553b509798869174 != \f\a\3\4\2\8\5\2\8\a\b\a\5\b\e\0\5\5\3\b\5\0\9\7\9\8\8\6\9\1\7\4 ]] 2024-09-18T00:31:53.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:31:53.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 82471 2024-09-18T00:31:53.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 82473 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 82475 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 82477 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 82479 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 82480 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 82482 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 82484 2024-09-18T00:31:53.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:31:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 82486 2024-09-18T00:31:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:31:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:31:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:31:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary1 2024-09-18T00:31:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:31:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:31:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:31:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+replaying 2024-09-18T00:31:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:31:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:53.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:31:53.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:31:53.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:31:53.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:31:53.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:31:53.515 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:31:54.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+replaying '' '' 2024-09-18T00:31:54.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:31:54.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:54.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:31:54.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:31:54.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:31:54.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:31:54.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:31:54.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:31:54.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:31:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:31:54.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: d18e1596-14e2-4da8-9e9a-daeec5ecfa30 2024-09-18T00:31:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:31:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":71114752.0,"last_snapshot_bytes":71114752,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619406,"remote_snapshot_timestamp":1726619406,"replay_state":"idle"} 2024-09-18T00:31:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:31:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:31:39 2024-09-18T00:31:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:31:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:31:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:31:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:31:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:31:54' 2024-09-18T00:31:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:31:54.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:31:54.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:31:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":71114752.0,"last_snapshot_bytes":71114752,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619406,"remote_snapshot_timestamp":1726619406,"replay_state":"idle"} 2024-09-18T00:31:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:31:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:31:54.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:31:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:31:54.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:31:54.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:31:54.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:31:54.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:31:54.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:31:39' 2024-09-18T00:31:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+replaying '2024-09-18 00:31:39' 2024-09-18T00:31:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:31:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:31:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:31:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:31:39' 2024-09-18T00:31:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:31:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:31:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:31:54.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d18e1596-14e2-4da8-9e9a-daeec5ecfa30up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":71114752.0,"last_snapshot_bytes":71114752,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619406,"remote_snapshot_timestamp":1726619406,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":126701568.0,"last_snapshot_bytes":126701568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619406,"remote_snapshot_timestamp":1726619406,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary3af284746-4562-4e55-b3a1-340279f4d151up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":124628992.0,"last_snapshot_bytes":124628992,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63795200.0,"last_snapshot_bytes":63795200,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60329984.0,"last_snapshot_bytes":60329984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619401,"remote_snapshot_timestamp":1726619401,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":61997056.0,"last_snapshot_bytes":61997056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54' 2024-09-18T00:31:54.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:31:54.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:31:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:31:39' 2024-09-18T00:31:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:31:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:31:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:31:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:31:54.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:31:54.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:31:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:31:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary1 2024-09-18T00:31:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:31:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:31:54.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary1 2024-09-18T00:31:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary1 2024-09-18T00:31:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:31:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary1 2024-09-18T00:31:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:31:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:31:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:31:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:31:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary1 2024-09-18T00:31:56.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:31:56.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary1 2024-09-18T00:31:56.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:31:56.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:31:56.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary2 2024-09-18T00:31:56.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:31:56.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:31:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:31:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+replaying 2024-09-18T00:31:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:31:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:31:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:31:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:31:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:31:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:31:56.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:31:57.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+replaying '' '' 2024-09-18T00:31:57.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:31:57.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:57.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:31:57.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:31:57.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:31:57.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:31:57.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:31:57.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:31:57.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:31:57.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:31:57.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 838864e7-c9c1-4cb7-9af3-e63a599ff8a4 2024-09-18T00:31:57.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:31:57.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":126701568.0,"last_snapshot_bytes":126701568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619406,"remote_snapshot_timestamp":1726619406,"replay_state":"idle"} 2024-09-18T00:31:57.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:31:57.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:31:39 2024-09-18T00:31:57.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:31:57.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:31:57.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:31:57.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:31:57.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:31:54' 2024-09-18T00:31:57.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:31:57.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:31:57.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:31:57.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":126701568.0,"last_snapshot_bytes":126701568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619406,"remote_snapshot_timestamp":1726619406,"replay_state":"idle"} 2024-09-18T00:31:57.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:31:57.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:31:57.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:31:57.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:31:57.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:31:57.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:31:57.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:31:57.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:31:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:31:39' 2024-09-18T00:31:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+replaying '2024-09-18 00:31:39' 2024-09-18T00:31:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:31:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:31:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:31:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:31:39' 2024-09-18T00:31:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:31:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:31:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:31:57.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary2838864e7-c9c1-4cb7-9af3-e63a599ff8a4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":126701568.0,"last_snapshot_bytes":126701568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619406,"remote_snapshot_timestamp":1726619406,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary3af284746-4562-4e55-b3a1-340279f4d151up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":124628992.0,"last_snapshot_bytes":124628992,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63795200.0,"last_snapshot_bytes":63795200,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60329984.0,"last_snapshot_bytes":60329984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619401,"remote_snapshot_timestamp":1726619401,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":61997056.0,"last_snapshot_bytes":61997056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:54' 2024-09-18T00:31:57.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:31:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:31:57.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:31:39' 2024-09-18T00:31:57.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:31:57.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:31:57.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:31:57.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:31:57.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:31:57.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:31:57.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:31:57.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary2 2024-09-18T00:31:57.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:31:57.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:57.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:31:57.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary2 2024-09-18T00:31:57.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary2 2024-09-18T00:31:58.009 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:31:58.006+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d697f65520 2024-09-18T00:31:58.009 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:31:58.006+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d697f656c0 2024-09-18T00:31:58.009 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:31:58.006+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d697f65930 2024-09-18T00:31:59.038 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:31:59.034+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:31:59.039 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:31:59.034+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:31:59.039 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:31:59.034+0000 7f849c00d640 -1 reset not still connected to 0x5629c65b2b60 2024-09-18T00:31:59.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:31:59.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary2 2024-09-18T00:31:59.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:31:59.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:59.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:31:59.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:31:59.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:31:59.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary2 2024-09-18T00:31:59.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:31:59.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary2 2024-09-18T00:31:59.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:31:59.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:31:59.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary3 2024-09-18T00:31:59.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:31:59.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:59.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:31:59.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:31:59.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+replaying 2024-09-18T00:31:59.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:31:59.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:31:59.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:31:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:31:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:31:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:31:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:31:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:32:00.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+replaying '' '' 2024-09-18T00:32:00.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:00.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:00.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:00.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:00.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:00.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:00.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:00.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:00.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:32:00.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:32:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: af284746-4562-4e55-b3a1-340279f4d151 2024-09-18T00:32:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:32:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":124628992.0,"last_snapshot_bytes":124628992,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"} 2024-09-18T00:32:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:32:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:31:39 2024-09-18T00:32:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:32:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:32:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:32:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:32:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:31:59' 2024-09-18T00:32:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:32:00.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:32:00.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:32:00.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":124628992.0,"last_snapshot_bytes":124628992,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"} 2024-09-18T00:32:00.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:32:00.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:00.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:32:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:32:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:32:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:32:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:32:00.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:31:39' 2024-09-18T00:32:00.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+replaying '2024-09-18 00:31:39' 2024-09-18T00:32:00.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:00.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:00.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:00.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:00.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:31:39' 2024-09-18T00:32:00.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:00.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:00.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:32:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK842544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary3af284746-4562-4e55-b3a1-340279f4d151up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":124628992.0,"last_snapshot_bytes":124628992,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63795200.0,"last_snapshot_bytes":63795200,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60329984.0,"last_snapshot_bytes":60329984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619401,"remote_snapshot_timestamp":1726619401,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":61997056.0,"last_snapshot_bytes":61997056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59' 2024-09-18T00:32:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:32:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:32:01.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:31:39' 2024-09-18T00:32:01.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:32:01.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:32:01.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:01.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:32:01.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:01.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:01.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:01.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary3 2024-09-18T00:32:01.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:32:01.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:01.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:01.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary3 2024-09-18T00:32:01.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary3 2024-09-18T00:32:02.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:32:02.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary3 2024-09-18T00:32:02.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:02.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:32:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary3 2024-09-18T00:32:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary3 2024-09-18T00:32:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:32:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary4 2024-09-18T00:32:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:32:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+replaying 2024-09-18T00:32:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:02.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:32:03.102 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:32:03.098+0000 7fac32fac640 -1 reset not still connected to 0x55a83e1c5040 2024-09-18T00:32:03.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+replaying '' '' 2024-09-18T00:32:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:03.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:32:03.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:32:03.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f28d710c-fce9-48ab-b72f-a7c632f6c1ff 2024-09-18T00:32:03.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:32:03.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"} 2024-09-18T00:32:03.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:32:03.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:31:39 2024-09-18T00:32:03.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:32:03.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:32:03.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:32:03.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:32:03.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:31:59' 2024-09-18T00:32:03.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:32:03.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:32:03.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:32:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"} 2024-09-18T00:32:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:32:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:03.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:32:03.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:03.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:32:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:32:03.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:32:03.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:32:03.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:31:39' 2024-09-18T00:32:03.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+replaying '2024-09-18 00:31:39' 2024-09-18T00:32:03.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:03.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:03.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:03.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:03.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:31:39' 2024-09-18T00:32:03.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:03.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:03.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:32:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary4f28d710c-fce9-48ab-b72f-a7c632f6c1ffup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63795200.0,"last_snapshot_bytes":63795200,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60329984.0,"last_snapshot_bytes":60329984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619401,"remote_snapshot_timestamp":1726619401,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":61997056.0,"last_snapshot_bytes":61997056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:31:59' 2024-09-18T00:32:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:32:03.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:32:03.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:31:39' 2024-09-18T00:32:03.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:32:03.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:32:03.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:03.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:32:03.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary4 2024-09-18T00:32:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:32:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:03.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:03.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary4 2024-09-18T00:32:03.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary4 2024-09-18T00:32:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:32:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary4 2024-09-18T00:32:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:05.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:05.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:32:05.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary4 2024-09-18T00:32:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary4 2024-09-18T00:32:05.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:05.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:32:05.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary5 2024-09-18T00:32:05.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:05.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:05.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:05.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:32:05.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+replaying 2024-09-18T00:32:05.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:05.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:05.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:05.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:05.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:05.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:05.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:05.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:32:06.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+replaying '' '' 2024-09-18T00:32:06.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:06.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:06.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:06.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:06.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:06.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:06.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:06.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:06.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:32:06.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:32:06.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4fd082f9-a5cb-42cc-9a45-3ebe1a50bbca 2024-09-18T00:32:06.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:32:06.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63795200.0,"last_snapshot_bytes":63795200,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"} 2024-09-18T00:32:06.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:32:06.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:31:39 2024-09-18T00:32:06.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:32:06.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:32:06.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:32:06.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:32:06.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:32:04' 2024-09-18T00:32:06.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:32:06.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:32:06.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:32:06.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63795200.0,"last_snapshot_bytes":63795200,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"} 2024-09-18T00:32:06.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:32:06.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:06.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:32:06.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:06.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:32:06.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:32:06.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:32:06.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:32:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:31:39' 2024-09-18T00:32:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+replaying '2024-09-18 00:31:39' 2024-09-18T00:32:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:31:39' 2024-09-18T00:32:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:32:06.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK642544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:04image-primary54fd082f9-a5cb-42cc-9a45-3ebe1a50bbcaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63795200.0,"last_snapshot_bytes":63795200,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:04image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:04image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60329984.0,"last_snapshot_bytes":60329984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:04image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619401,"remote_snapshot_timestamp":1726619401,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:04image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":61997056.0,"last_snapshot_bytes":61997056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:31:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:04' 2024-09-18T00:32:06.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:32:06.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:32:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:31:39' 2024-09-18T00:32:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:32:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:32:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:06.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:32:06.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:06.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:06.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:06.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary5 2024-09-18T00:32:06.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:32:06.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:06.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:06.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary5 2024-09-18T00:32:07.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary5 2024-09-18T00:32:07.986 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:32:07.982+0000 7fac32fac640 -1 reset not still connected to 0x55a83a76a8f0 2024-09-18T00:32:07.986 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:32:07.982+0000 7fac32fac640 -1 reset not still connected to 0x55a83e1c5040 2024-09-18T00:32:08.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:32:08.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary5 2024-09-18T00:32:08.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:08.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:08.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:08.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:08.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:32:08.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary5 2024-09-18T00:32:08.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:08.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary5 2024-09-18T00:32:08.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:08.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:32:08.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary6 2024-09-18T00:32:08.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:08.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:08.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:08.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:32:08.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+replaying 2024-09-18T00:32:08.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:08.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:08.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:08.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:08.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:08.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:08.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:08.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:32:09.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+replaying '' '' 2024-09-18T00:32:09.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:09.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:09.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:09.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:09.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:09.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:09.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:09.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:09.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:32:09.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:32:09.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: b713c9d1-2c92-4cf8-9e0a-982f46d2d359 2024-09-18T00:32:09.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:32:09.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"} 2024-09-18T00:32:09.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:32:09.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:32:09 2024-09-18T00:32:09.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:32:09.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:32:09.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:32:09.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:32:09.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:32:09' 2024-09-18T00:32:09.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:32:09.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:32:09.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:32:09.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"} 2024-09-18T00:32:09.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:32:09.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:09.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:32:09.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:09.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:32:09.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:32:09.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:32:09.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:32:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:32:09' 2024-09-18T00:32:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+replaying '2024-09-18 00:32:09' 2024-09-18T00:32:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:32:09' 2024-09-18T00:32:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:32:09.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:09image-primary6b713c9d1-2c92-4cf8-9e0a-982f46d2d359up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70234112.0,"last_snapshot_bytes":70234112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619397,"remote_snapshot_timestamp":1726619397,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:09image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60329984.0,"last_snapshot_bytes":60329984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:09image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619401,"remote_snapshot_timestamp":1726619401,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:09image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":61997056.0,"last_snapshot_bytes":61997056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:09' 2024-09-18T00:32:09.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:32:09.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:32:09.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:32:09' 2024-09-18T00:32:09.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:32:09.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:32:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:32:09.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:09.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:09.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:09.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary6 2024-09-18T00:32:09.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:32:09.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:09.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:09.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary6 2024-09-18T00:32:09.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary6 2024-09-18T00:32:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:32:11.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary6 2024-09-18T00:32:11.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:11.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:11.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:11.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:11.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:32:11.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary6 2024-09-18T00:32:11.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:11.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary6 2024-09-18T00:32:11.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary7 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+replaying 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:11.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:11.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:11.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:32:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+replaying '' '' 2024-09-18T00:32:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:12.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:12.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:32:12.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:32:12.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 068a799c-59b8-4a9a-acb8-c91e7357896b 2024-09-18T00:32:12.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:32:12.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60329984.0,"last_snapshot_bytes":60329984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"} 2024-09-18T00:32:12.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:32:12.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:32:09 2024-09-18T00:32:12.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:32:12.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:32:12.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:32:12.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:32:12.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:32:09' 2024-09-18T00:32:12.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:32:12.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:32:12.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:32:12.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60329984.0,"last_snapshot_bytes":60329984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"} 2024-09-18T00:32:12.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:32:12.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:12.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:32:12.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:12.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:32:12.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:32:12.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:32:12.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:32:12.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:32:09' 2024-09-18T00:32:12.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+replaying '2024-09-18 00:32:09' 2024-09-18T00:32:12.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:12.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:12.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:12.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:12.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:32:09' 2024-09-18T00:32:12.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:12.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:12.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:32:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:09image-primary7068a799c-59b8-4a9a-acb8-c91e7357896bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60329984.0,"last_snapshot_bytes":60329984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619398,"remote_snapshot_timestamp":1726619398,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:09image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619401,"remote_snapshot_timestamp":1726619401,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:09image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":61997056.0,"last_snapshot_bytes":61997056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:09' 2024-09-18T00:32:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:32:12.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:32:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:32:09' 2024-09-18T00:32:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:32:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:32:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:32:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:12.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary7 2024-09-18T00:32:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:32:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:12.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary7 2024-09-18T00:32:12.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary7 2024-09-18T00:32:13.599 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:32:13.598+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:32:14.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:32:14.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary7 2024-09-18T00:32:14.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:14.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:14.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:14.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:14.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:32:14.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary7 2024-09-18T00:32:14.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:14.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary7 2024-09-18T00:32:14.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:14.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:32:14.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary8 2024-09-18T00:32:14.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+replaying 2024-09-18T00:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:32:15.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+replaying '' '' 2024-09-18T00:32:15.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:15.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:15.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:15.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:15.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:15.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:15.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:15.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:15.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:32:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:32:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 51c15641-cc65-4aa0-a9cf-9e4d6425d92c 2024-09-18T00:32:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:32:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619401,"remote_snapshot_timestamp":1726619401,"replay_state":"idle"} 2024-09-18T00:32:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:32:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:32:09 2024-09-18T00:32:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:32:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:32:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:32:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:32:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:32:14' 2024-09-18T00:32:15.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:32:15.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:32:15.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:32:15.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619401,"remote_snapshot_timestamp":1726619401,"replay_state":"idle"} 2024-09-18T00:32:15.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:32:15.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:15.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:32:15.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:15.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:32:15.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:32:15.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:32:15.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:32:15.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:32:09' 2024-09-18T00:32:15.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+replaying '2024-09-18 00:32:09' 2024-09-18T00:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:32:09' 2024-09-18T00:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:15.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:15.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:32:15.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK342544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:14image-primary851c15641-cc65-4aa0-a9cf-9e4d6425d92cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70057984.0,"last_snapshot_bytes":70057984,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619401,"remote_snapshot_timestamp":1726619401,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:14image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":61997056.0,"last_snapshot_bytes":61997056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:14' 2024-09-18T00:32:15.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:32:15.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:32:15.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:32:09' 2024-09-18T00:32:15.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:32:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:32:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:15.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:32:15.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:15.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:15.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:15.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary8 2024-09-18T00:32:15.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:32:15.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:15.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:15.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary8 2024-09-18T00:32:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary8 2024-09-18T00:32:15.910 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:32:15.905+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b040 2024-09-18T00:32:15.913 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:32:15.909+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b380 2024-09-18T00:32:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:32:17.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary8 2024-09-18T00:32:17.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:32:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary8 2024-09-18T00:32:17.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:17.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary8 2024-09-18T00:32:17.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:32:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary9 2024-09-18T00:32:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:32:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+replaying 2024-09-18T00:32:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:17.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:17.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:17.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:32:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+replaying '' '' 2024-09-18T00:32:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:18.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:18.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:18.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:32:18.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:32:18.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a4a49402-ee8f-4527-b9d9-e3c5d7289204 2024-09-18T00:32:18.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:32:18.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":61997056.0,"last_snapshot_bytes":61997056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"} 2024-09-18T00:32:18.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:32:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:32:09 2024-09-18T00:32:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:32:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:32:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:32:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:32:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:32:14' 2024-09-18T00:32:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:32:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:32:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:32:18.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":61997056.0,"last_snapshot_bytes":61997056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"} 2024-09-18T00:32:18.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:32:18.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:18.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:32:18.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:18.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:32:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:32:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:32:18.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:32:18.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:32:09' 2024-09-18T00:32:18.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+replaying '2024-09-18 00:32:09' 2024-09-18T00:32:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:32:09' 2024-09-18T00:32:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:32:18.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:14image-primary9a4a49402-ee8f-4527-b9d9-e3c5d7289204up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":61997056.0,"last_snapshot_bytes":61997056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:14' 2024-09-18T00:32:18.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:32:18.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:32:18.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:32:09' 2024-09-18T00:32:18.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:32:18.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:32:18.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:18.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:32:18.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:18.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:18.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:18.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary9 2024-09-18T00:32:18.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:32:18.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:18.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:18.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary9 2024-09-18T00:32:18.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary9 2024-09-18T00:32:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:32:20.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary9 2024-09-18T00:32:20.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:20.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:20.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:20.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:20.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:32:20.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary9 2024-09-18T00:32:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:20.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary9 2024-09-18T00:32:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary10 2024-09-18T00:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+replaying 2024-09-18T00:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:32:21.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+replaying '' '' 2024-09-18T00:32:21.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:21.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:21.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:21.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:21.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:32:21.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:32:21.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:21.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:21.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:32:21.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:32:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4ce86fed-82d5-428f-affa-37bf7b004ee8 2024-09-18T00:32:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:32:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"} 2024-09-18T00:32:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:32:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:32:09 2024-09-18T00:32:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:32:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:32:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:32:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:32:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:32:19' 2024-09-18T00:32:21.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:32:21.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:32:21.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:32:21.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"} 2024-09-18T00:32:21.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:32:21.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:21.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:32:21.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:21.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:32:21.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:32:21.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:32:21.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:32:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:32:09' 2024-09-18T00:32:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+replaying '2024-09-18 00:32:09' 2024-09-18T00:32:21.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:21.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:21.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:21.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:32:21.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:32:09' 2024-09-18T00:32:21.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:32:21.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:32:21.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:32:21.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary104ce86fed-82d5-428f-affa-37bf7b004ee8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619403,"remote_snapshot_timestamp":1726619403,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:32:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:32:19' 2024-09-18T00:32:21.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:32:21.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:32:21.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:32:09' 2024-09-18T00:32:21.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:32:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:32:21.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:32:21.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary10 2024-09-18T00:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary10 2024-09-18T00:32:21.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary10 2024-09-18T00:32:23.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:32:23.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary10 2024-09-18T00:32:23.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:23.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:23.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:23.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:32:23.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:32:23.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary10 2024-09-18T00:32:23.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:23.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary10 2024-09-18T00:32:23.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..10} 2024-09-18T00:32:23.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS=() 2024-09-18T00:32:23.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS=() 2024-09-18T00:32:23.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS=() 2024-09-18T00:32:23.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS=() 2024-09-18T00:32:23.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RET=0 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary1 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary1 snapshot 10G 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:23.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary1 10G 2024-09-18T00:32:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:32:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:32:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:32:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:32:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary1 2024-09-18T00:32:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:32:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:32:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary1 snapshot 2024-09-18T00:32:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:32:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:23.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary1 snapshot 2024-09-18T00:32:24.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:32:24.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary1 2024-09-18T00:32:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary1': 2024-09-18T00:32:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:32:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:32:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:32:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 272d7b3807db 2024-09-18T00:32:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.272d7b3807db 2024-09-18T00:32:24.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:32:24.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:32:24.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:32:24.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:32:24.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:32:23 2024 2024-09-18T00:32:24.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:32:23 2024 2024-09-18T00:32:24.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:32:23 2024 2024-09-18T00:32:24.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:32:24.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:32:24.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 77cc37a8-1548-46a1-ac79-91a5d3172bb9 2024-09-18T00:32:24.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:32:24.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:32:24.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:32:25.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd0 2024-09-18T00:32:25.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:32:25.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd0 2024-09-18T00:32:25.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:32:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:32:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:32:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 318c6cae-01ee-4b35-bd39-7ab765e41e4e 2024-09-18T00:32:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:32:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:32:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:25.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:32:25.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:32:25.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:32:25.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:32:25.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:25.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary1 2024-09-18T00:32:25.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd0 test-primary1 2024-09-18T00:32:26.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:32:26.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary1 2024-09-18T00:32:26.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:32:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary1 2024-09-18T00:32:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:32:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary1 2024-09-18T00:32:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:32:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary2 2024-09-18T00:32:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary2 snapshot 10G 2024-09-18T00:32:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:32:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary1 2024-09-18T00:32:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:32:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:32:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary2 10G 2024-09-18T00:32:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary1 2024-09-18T00:32:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:32:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:32:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:32:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:32:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:32:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:26.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary2 2024-09-18T00:32:26.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:26.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:32:26.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:32:26.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:32:26.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:26.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:32:26.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:26.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary2 snapshot 2024-09-18T00:32:26.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:26.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:26.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:32:26.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:26.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary2 snapshot 2024-09-18T00:32:26.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1656 2024-09-18T00:32:26.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:32:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:26.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary2 2024-09-18T00:32:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary2': 2024-09-18T00:32:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:32:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:32:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:32:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 273c598940b8 2024-09-18T00:32:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.273c598940b8 2024-09-18T00:32:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:32:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:32:26.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:32:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:32:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:32:26 2024 2024-09-18T00:32:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:32:26 2024 2024-09-18T00:32:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:32:26 2024 2024-09-18T00:32:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:32:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:32:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: f5aaab3c-2eab-452d-b16b-6eb190381b85 2024-09-18T00:32:26.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:32:26.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:32:26.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:32:27.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd1 2024-09-18T00:32:27.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:32:27.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd1 2024-09-18T00:32:27.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:32:27.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:32:27.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary1/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary1' 2024-09-18T00:32:27.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:32:27.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:32:27.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: c63a1959-8f08-4b2e-b07e-61383e502dc6 2024-09-18T00:32:27.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:32:27.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:32:27.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:27.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:32:27.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:32:27.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:32:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:32:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary2 2024-09-18T00:32:27.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd1 test-primary2 2024-09-18T00:32:27.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:32:27.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary2 2024-09-18T00:32:27.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:32:27.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary2 2024-09-18T00:32:27.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:27.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:27.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:32:27.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:27.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:32:27.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:32:27.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary2 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary2 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary2 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary3 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary3 snapshot 10G 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:27.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary3 10G 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:32:27.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:32:27.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:27.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary3 2024-09-18T00:32:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:32:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:32:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary3 snapshot 2024-09-18T00:32:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary3 snapshot 2024-09-18T00:32:27.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:32:27.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary2/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary2' 2024-09-18T00:32:28.052 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:32:28.049+0000 7fac32fac640 -1 reset not still connected to 0x55a839605ba0 2024-09-18T00:32:28.053 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:32:28.049+0000 7fac32fac640 -1 reset not still connected to 0x55a83a5192b0 2024-09-18T00:32:28.053 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:32:28.049+0000 7fac32fac640 -1 reset not still connected to 0x55a83a519e10 2024-09-18T00:32:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:32:28.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1658 2024-09-18T00:32:28.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:28.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary3 2024-09-18T00:32:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary3': 2024-09-18T00:32:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:32:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:32:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:32:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 274b52448734 2024-09-18T00:32:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.274b52448734 2024-09-18T00:32:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:32:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:32:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:32:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:32:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:32:27 2024 2024-09-18T00:32:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:32:27 2024 2024-09-18T00:32:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:32:27 2024 2024-09-18T00:32:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:32:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:32:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 70ed3363-f67f-4aab-a4f4-2e9dca8257a3 2024-09-18T00:32:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:32:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:32:28.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:32:29.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd2 2024-09-18T00:32:29.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:32:29.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd2 2024-09-18T00:32:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:32:29.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:32:29.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:32:29.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 5299cfdd-10a0-4eb6-afbc-b1c8c9133c3d 2024-09-18T00:32:29.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:32:29.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:32:29.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:29.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:32:29.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:32:29.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:32:29.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:32:29.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:29.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary3 2024-09-18T00:32:29.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd2 test-primary3 2024-09-18T00:32:29.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:32:29.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary3 2024-09-18T00:32:29.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:32:29.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary3 2024-09-18T00:32:29.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:29.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:29.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:29.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:29.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:32:29.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary3 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary4 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary4 snapshot 10G 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary3 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:29.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:29.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:29.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:29.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:29.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary3 2024-09-18T00:32:29.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:29.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:32:29.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:29.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary4 10G 2024-09-18T00:32:29.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:29.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:29.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:29.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:29.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:29.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:29.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:32:29.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:32:29.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:32:29.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:29.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary4 2024-09-18T00:32:29.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:29.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:32:29.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:29.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:29.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:32:29.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:32:29.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:32:29.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:29.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:32:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary4 snapshot 2024-09-18T00:32:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:29.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary4 snapshot 2024-09-18T00:32:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:32:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary3/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary3' 2024-09-18T00:32:30.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:32:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1662 2024-09-18T00:32:30.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary4 2024-09-18T00:32:30.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:30.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary4': 2024-09-18T00:32:30.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:32:30.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:32:30.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:32:30.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 275c8a5e6aa1 2024-09-18T00:32:30.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.275c8a5e6aa1 2024-09-18T00:32:30.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:32:30.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:32:30.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:32:30.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:32:30.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:32:29 2024 2024-09-18T00:32:30.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:32:29 2024 2024-09-18T00:32:30.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:32:29 2024 2024-09-18T00:32:30.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:32:30.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:32:30.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 732efaab-7893-47c3-85bb-e14c29d2c94f 2024-09-18T00:32:30.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:32:30.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:32:30.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:32:31.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd3 2024-09-18T00:32:31.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:32:31.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd3 2024-09-18T00:32:31.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:32:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:32:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:32:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: d4501379-2127-4701-b2fa-870573c0f85f 2024-09-18T00:32:31.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:32:31.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:32:31.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:31.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:32:31.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:32:31.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:32:31.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:32:31.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:31.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary4 2024-09-18T00:32:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd3 test-primary4 2024-09-18T00:32:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:32:31.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary4 2024-09-18T00:32:31.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:32:31.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary4 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary5 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary4 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary5 snapshot 10G 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary5 10G 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary4 2024-09-18T00:32:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary4 2024-09-18T00:32:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:32:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:32:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:32:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary5 2024-09-18T00:32:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:32:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:32:31.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary5 snapshot 2024-09-18T00:32:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:31.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:31.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:31.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:31.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary5 snapshot 2024-09-18T00:32:31.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:32:31.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary4/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary4' 2024-09-18T00:32:31.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1660 2024-09-18T00:32:31.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:31.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:31.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:32:31.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:31.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:31.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:32:31.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:32:32.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1663 2024-09-18T00:32:32.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:32:32.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:32.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary5 2024-09-18T00:32:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary5': 2024-09-18T00:32:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:32:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:32:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:32:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 276ee31b1b0d 2024-09-18T00:32:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.276ee31b1b0d 2024-09-18T00:32:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:32:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:32:32.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:32:32.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:32:32.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:32:31 2024 2024-09-18T00:32:32.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:32:31 2024 2024-09-18T00:32:32.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:32:31 2024 2024-09-18T00:32:32.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:32:32.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:32:32.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 0dcf2296-c094-48fc-b25c-a438556ec60f 2024-09-18T00:32:32.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:32:32.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:32:32.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:32:32.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd4 2024-09-18T00:32:32.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:32:32.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd4 2024-09-18T00:32:32.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:32:32.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:32:32.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:32:32.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: b8c5218f-8cac-4930-b8bd-4f86b9f9a5c8 2024-09-18T00:32:32.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:32:32.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:32:32.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:32.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:32:32.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:32:33.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:32:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:32:33.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:33.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary5 2024-09-18T00:32:33.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd4 test-primary5 2024-09-18T00:32:33.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1665 2024-09-18T00:32:33.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:33.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:32:33.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary5 2024-09-18T00:32:33.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:32:33.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary5 2024-09-18T00:32:33.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary5 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:32:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary6 2024-09-18T00:32:33.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary6 snapshot 10G 2024-09-18T00:32:33.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary5 2024-09-18T00:32:33.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary5 2024-09-18T00:32:33.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:33.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:33.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:33.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:33.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:33.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:33.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary6 10G 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:33.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary6 2024-09-18T00:32:33.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:32:33.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:33.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:32:33.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:33.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary6 snapshot 2024-09-18T00:32:33.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:33.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:33.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:33.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:33.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary6 snapshot 2024-09-18T00:32:33.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:32:33.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary5/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary5' 2024-09-18T00:32:33.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:33.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:32:33.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:33.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:33.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:33.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:32:34.449 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:32:34.445+0000 7fac32fac640 -1 reset not still connected to 0x55a839605ba0 2024-09-18T00:32:34.449 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:32:34.445+0000 7fac32fac640 -1 reset not still connected to 0x55a83a5192b0 2024-09-18T00:32:34.449 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:32:34.445+0000 7fac32fac640 -1 reset not still connected to 0x55a83a519e10 2024-09-18T00:32:34.449 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:32:34.445+0000 7fac32fac640 -1 reset not still connected to 0x55a83cc06820 2024-09-18T00:32:34.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1666 2024-09-18T00:32:34.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:34.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:32:34.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary6 2024-09-18T00:32:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary6': 2024-09-18T00:32:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:32:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:32:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:32:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2780a133b055 2024-09-18T00:32:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2780a133b055 2024-09-18T00:32:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:32:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:32:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:32:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:32:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:32:33 2024 2024-09-18T00:32:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:32:33 2024 2024-09-18T00:32:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:32:33 2024 2024-09-18T00:32:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:32:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:32:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 7e38e8df-a146-46c4-95b7-7e5c162fb6b5 2024-09-18T00:32:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:32:34.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:32:34.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:32:34.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:34.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:32:34.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:34.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:34.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:32:34.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:32:34.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd5 2024-09-18T00:32:34.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:32:34.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd5 2024-09-18T00:32:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:32:34.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:32:34.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:32:34.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 96cc68f4-0fe6-41e3-b4b6-35b8f1f8438c 2024-09-18T00:32:34.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:32:34.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:32:34.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:34.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:32:34.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:32:35.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:32:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:32:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:35.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary6 2024-09-18T00:32:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd5 test-primary6 2024-09-18T00:32:35.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1668 2024-09-18T00:32:35.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:32:35.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary6 2024-09-18T00:32:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:32:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary6 2024-09-18T00:32:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:32:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary6 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary7 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary7 snapshot 10G 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:35.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary6 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary6 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary7 10G 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:35.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:35.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:35.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:35.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:32:35.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:32:35.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:32:35.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:35.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary7 2024-09-18T00:32:35.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:35.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:35.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:32:35.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:35.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:35.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:35.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:32:35.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:32:35.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:35.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:32:35.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:35.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary7 snapshot 2024-09-18T00:32:35.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:35.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:35.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:35.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:35.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary7 snapshot 2024-09-18T00:32:35.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:32:35.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary6/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary6' 2024-09-18T00:32:36.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1669 2024-09-18T00:32:36.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:36.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:36.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:32:36.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:36.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:36.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:32:36.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:32:36.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1670 2024-09-18T00:32:36.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:36.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:32:36.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary7 2024-09-18T00:32:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary7': 2024-09-18T00:32:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:32:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:32:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:32:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2793410c3c34 2024-09-18T00:32:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2793410c3c34 2024-09-18T00:32:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:32:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:32:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:32:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:32:36.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:32:35 2024 2024-09-18T00:32:36.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:32:35 2024 2024-09-18T00:32:36.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:32:35 2024 2024-09-18T00:32:36.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:32:36.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:32:36.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: d8d7d52c-37d9-4ee7-a7b4-40649be59525 2024-09-18T00:32:36.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:32:36.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:32:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:32:36.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd6 2024-09-18T00:32:36.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:32:36.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd6 2024-09-18T00:32:36.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:32:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:32:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:32:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: a1e20eed-8b96-4ede-a0fb-99ab3367464b 2024-09-18T00:32:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:32:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:32:36.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:36.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:32:36.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:32:37.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:32:37.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:32:37.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:37.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary7 2024-09-18T00:32:37.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd6 test-primary7 2024-09-18T00:32:37.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:32:37.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary7 2024-09-18T00:32:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:32:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary7 2024-09-18T00:32:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:32:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:32:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary7 2024-09-18T00:32:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:32:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:32:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary8 2024-09-18T00:32:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary8 snapshot 10G 2024-09-18T00:32:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary7 2024-09-18T00:32:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary7 2024-09-18T00:32:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:32:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary8 10G 2024-09-18T00:32:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:37.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:32:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:32:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:32:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:37.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary8 2024-09-18T00:32:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:32:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary7/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary7' 2024-09-18T00:32:37.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:32:37.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:37.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:32:37.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:37.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary8 snapshot 2024-09-18T00:32:37.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:37.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:37.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:37.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:37.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary8 snapshot 2024-09-18T00:32:37.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1671 2024-09-18T00:32:37.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:37.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:37.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:32:37.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:37.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:37.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:37.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:32:38.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1673 2024-09-18T00:32:38.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1674 2024-09-18T00:32:38.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:38.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:32:38.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:38.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:38.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:38.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:32:38.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:38.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:38.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:32:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary8 2024-09-18T00:32:38.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary8': 2024-09-18T00:32:38.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:32:38.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:32:38.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:32:38.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 27a9734a9d 2024-09-18T00:32:38.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.27a9734a9d 2024-09-18T00:32:38.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:32:38.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:32:38.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:32:38.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:32:38.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:32:37 2024 2024-09-18T00:32:38.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:32:37 2024 2024-09-18T00:32:38.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:32:37 2024 2024-09-18T00:32:38.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:32:38.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:32:38.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: a8d82560-37fd-4dc8-999f-53c47be6dcd9 2024-09-18T00:32:38.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:32:38.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:32:38.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:32:39.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd7 2024-09-18T00:32:39.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:32:39.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd7 2024-09-18T00:32:39.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:32:39.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:32:39.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:32:39.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: a0dc5bdb-e151-41ab-b709-057cad2fb0a6 2024-09-18T00:32:39.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:32:39.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:32:39.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:39.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:32:39.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:32:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:32:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:32:39.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:32:39.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:39.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:32:39.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:39.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:39.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:39.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:32:39.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:32:39.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 Snapshot ID: 1676 2024-09-18T00:32:39.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:39.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:32:39.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:39.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary8 2024-09-18T00:32:39.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd7 test-primary8 2024-09-18T00:32:39.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:32:39.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary8 2024-09-18T00:32:39.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:32:39.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary8 2024-09-18T00:32:39.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:32:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:32:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary9 2024-09-18T00:32:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:32:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary9 snapshot 10G 2024-09-18T00:32:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:39.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:32:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:32:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary9 10G 2024-09-18T00:32:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:32:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:32:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:32:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary9 2024-09-18T00:32:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary8 2024-09-18T00:32:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary8 2024-09-18T00:32:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary8 2024-09-18T00:32:40.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:32:40.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary8/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary8' 2024-09-18T00:32:40.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:32:40.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:40.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:32:40.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:40.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary9 snapshot 2024-09-18T00:32:40.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:40.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:40.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:40.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:40.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary9 snapshot 2024-09-18T00:32:40.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1677 2024-09-18T00:32:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:32:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:40.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:40.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:32:40.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:40.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1680 2024-09-18T00:32:40.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:40.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:32:40.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary9 2024-09-18T00:32:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary9': 2024-09-18T00:32:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:32:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:32:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:32:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 27c1c465054b 2024-09-18T00:32:40.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.27c1c465054b 2024-09-18T00:32:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:32:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:32:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:32:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:32:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:32:40 2024 2024-09-18T00:32:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:32:40 2024 2024-09-18T00:32:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:32:40 2024 2024-09-18T00:32:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:32:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:32:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: a3cef58b-69ac-4f11-993d-23070a4b6c51 2024-09-18T00:32:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:32:40.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:32:40.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:32:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd8 2024-09-18T00:32:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:32:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd8 2024-09-18T00:32:41.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:32:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:32:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:32:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: ce6e676a-8411-4ad3-a740-dd13c6d0e2aa 2024-09-18T00:32:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:32:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:32:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:32:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:32:41.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): Snapshot ID: 1679 2024-09-18T00:32:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:32:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:32:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:32:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:32:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:32:41.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:41.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1678 2024-09-18T00:32:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:41.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:32:41.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:32:41.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:41.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary9 2024-09-18T00:32:41.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd8 test-primary9 2024-09-18T00:32:42.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:32:42.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary9 2024-09-18T00:32:42.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary10 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary10 snapshot 10G 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary10 10G 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary10 2024-09-18T00:32:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary9 2024-09-18T00:32:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary9 2024-09-18T00:32:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary9 2024-09-18T00:32:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary9 2024-09-18T00:32:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:32:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:42.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:32:42.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:32:42.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:32:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:32:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary10 snapshot 2024-09-18T00:32:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:32:42.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary10 snapshot 2024-09-18T00:32:42.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:32:42.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary9/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary9' 2024-09-18T00:32:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1682 2024-09-18T00:32:42.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:42.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1685 2024-09-18T00:32:42.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:32:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:32:43.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:43.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:32:43.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:43.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:43.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:43.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:32:43.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1683 2024-09-18T00:32:43.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1684 2024-09-18T00:32:43.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:43.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:43.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:32:43.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:43.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:32:43.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:43.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:43.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:43.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:32:43.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary10 2024-09-18T00:32:43.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary10': 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 27d94aaca209 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.27d94aaca209 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:32:42 2024 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:32:42 2024 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:32:42 2024 2024-09-18T00:32:43.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:32:43.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:32:43.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: e4321661-07df-4164-962e-0d882a261feb 2024-09-18T00:32:43.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:32:43.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:32:43.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:32:43.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd9 2024-09-18T00:32:43.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:32:43.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd9 2024-09-18T00:32:43.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:32:44.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 524288/2621440 done 2024-09-18T00:32:44.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:32:44.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 1c6c85ea-d67c-4fad-a154-ce3cc2da296a 2024-09-18T00:32:44.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:32:44.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:32:44.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:44.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:32:44.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:32:44.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:32:44.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:44.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:32:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:32:44.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 Snapshot ID: 1687 2024-09-18T00:32:44.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:32:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:32:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:32:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:32:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:32:44.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary10 2024-09-18T00:32:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd9 test-primary10 2024-09-18T00:32:44.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:32:44.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary10 2024-09-18T00:32:44.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:32:44.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary10 2024-09-18T00:32:44.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:44.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:32:44.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:44.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:44.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:44.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary10 2024-09-18T00:32:44.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:32:44.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:32:44.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 86154 2024-09-18T00:32:44.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:44.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:44.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:44.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:32:44.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary10 2024-09-18T00:32:44.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary10 2024-09-18T00:32:44.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:32:44.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary10/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary10' 2024-09-18T00:32:45.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:45.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:32:45.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:45.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:45.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:45.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:32:45.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1689 2024-09-18T00:32:45.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:45.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1688 2024-09-18T00:32:45.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:32:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:32:45.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1692 2024-09-18T00:32:45.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:46.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1690 2024-09-18T00:32:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:32:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:46.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:32:46.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:46.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1691 2024-09-18T00:32:46.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:46.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:32:46.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:46.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:46.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:32:46.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:32:46.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:47.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1693 2024-09-18T00:32:47.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:47.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1694 2024-09-18T00:32:47.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:47.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:47.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:32:47.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:47.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:47.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:47.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:32:48.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1695 2024-09-18T00:32:48.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:48.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1696 2024-09-18T00:32:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:48.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:48.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:32:48.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:48.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:48.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:48.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:32:48.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:48.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:32:48.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:48.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:48.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:48.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:32:48.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:48.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:32:48.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:48.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:48.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:48.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:32:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1697 2024-09-18T00:32:49.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:32:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:32:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:32:49.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:49.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:32:49.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:49.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:49.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:49.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:32:50.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:50.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:32:50.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:50.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:50.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:50.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:32:50.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1698 2024-09-18T00:32:50.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1700 2024-09-18T00:32:50.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1699 2024-09-18T00:32:50.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:50.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:32:50.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:50.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:50.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:50.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:32:50.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:50.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:50.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:32:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:51.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:32:51.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:51.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:32:51.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:51.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:51.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:32:51.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:32:51.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1701 2024-09-18T00:32:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:51.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1702 2024-09-18T00:32:51.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:52.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1703 2024-09-18T00:32:52.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1704 2024-09-18T00:32:52.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:32:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:52.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:32:53.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:53.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:32:53.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:53.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:53.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:53.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:32:53.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:53.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:32:53.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:53.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:53.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:53.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:32:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1705 2024-09-18T00:32:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:32:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:32:53.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1706 2024-09-18T00:32:53.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:53.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:54.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1707 2024-09-18T00:32:54.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:54.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:32:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:32:54.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:32:54.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:54.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:32:54.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:54.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:54.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:54.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:32:55.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:55.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:32:55.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:55.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:55.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:32:55.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:32:55.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:55.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:32:55.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:55.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:55.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:32:55.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:32:55.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1709 2024-09-18T00:32:55.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1708 2024-09-18T00:32:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1710 2024-09-18T00:32:55.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:55.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:56.450 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:32:56.440+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f89c0 2024-09-18T00:32:56.450 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:32:56.440+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T00:32:56.451 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:32:56.440+0000 7f52d7c6c640 -1 reset not still connected to 0x557848682270 2024-09-18T00:32:56.451 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:32:56.440+0000 7f52d7c6c640 -1 reset not still connected to 0x5578486f5860 2024-09-18T00:32:56.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1712 2024-09-18T00:32:56.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:56.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:32:56.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:56.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:56.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:32:56.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:32:56.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:56.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:32:56.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:56.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:56.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1711 2024-09-18T00:32:56.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:32:56.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:32:56.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:56.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:57.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1713 2024-09-18T00:32:57.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1714 2024-09-18T00:32:57.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:57.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:57.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:32:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:32:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:32:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1715 2024-09-18T00:32:58.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:58.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:32:58.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:58.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:58.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:32:58.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:32:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:32:58.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:58.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:58.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:32:58.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:32:58.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1716 2024-09-18T00:32:58.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:58.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:32:58.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:58.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:32:58.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:58.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:58.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:32:58.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:32:59.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:59.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:32:59.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:59.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:59.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:32:59.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:32:59.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1717 2024-09-18T00:32:59.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:32:59.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:32:59.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:32:59.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:32:59.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:32:59.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:32:59.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:00.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:00.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:00.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:00.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:00.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:00.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:00.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:00.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:00.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:00.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:00.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:00.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:00.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1720 2024-09-18T00:33:00.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1719 2024-09-18T00:33:00.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:00.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1718 2024-09-18T00:33:00.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:00.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:01.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:01.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:33:01.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:01.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:01.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:33:01.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:33:01.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1721 2024-09-18T00:33:01.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1722 2024-09-18T00:33:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:33:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:01.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:01.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:33:01.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:33:01.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:01.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:02.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1723 2024-09-18T00:33:02.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1724 2024-09-18T00:33:02.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:02.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:02.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:02.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:33:02.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:02.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:02.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:33:02.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:33:03.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1725 2024-09-18T00:33:03.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1726 2024-09-18T00:33:03.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:03.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:33:03.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:03.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:03.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:33:03.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:33:03.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:03.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:03.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:33:03.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:03.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:03.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:33:03.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:33:03.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:03.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:03.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:33:03.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:03.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:03.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:33:03.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:33:04.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1727 2024-09-18T00:33:04.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:04.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:04.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:33:04.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:04.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:04.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:33:04.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:33:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:33:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:33:04.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:33:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:05.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1729 2024-09-18T00:33:05.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1730 2024-09-18T00:33:05.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1728 2024-09-18T00:33:05.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:05.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:06.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:06.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:33:06.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:33:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:33:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1731 2024-09-18T00:33:06.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1732 2024-09-18T00:33:06.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:06.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:33:06.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:06.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:06.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:33:06.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:33:06.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:06.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:07.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1734 2024-09-18T00:33:07.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1733 2024-09-18T00:33:07.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:07.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:07.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:07.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:33:07.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:33:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:33:08.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1736 2024-09-18T00:33:08.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1735 2024-09-18T00:33:08.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:08.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:08.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:33:08.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:08.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:08.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:33:08.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:33:08.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:08.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:08.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:33:08.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:08.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:08.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:33:08.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:33:08.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:08.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:33:08.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:08.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:08.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:33:08.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:33:09.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1737 2024-09-18T00:33:09.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:09.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:33:09.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:09.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:09.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:33:09.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:33:09.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:09.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:33:09.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:09.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:09.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:33:09.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:33:09.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:10.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:10.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:10.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:10.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:10.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:10.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:10.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1739 2024-09-18T00:33:10.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1740 2024-09-18T00:33:10.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:10.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1738 2024-09-18T00:33:10.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:10.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:11.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:11.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:33:11.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:11.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:11.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:33:11.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:33:11.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:11.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:33:11.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:11.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:11.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:33:11.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:33:11.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1742 2024-09-18T00:33:11.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1741 2024-09-18T00:33:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1744 2024-09-18T00:33:12.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1743 2024-09-18T00:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:12.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:33:12.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:12.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:12.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:33:12.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:33:13.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1745 2024-09-18T00:33:13.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1746 2024-09-18T00:33:13.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:13.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:33:13.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:13.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:13.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:33:13.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:33:13.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:13.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:13.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:33:13.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:13.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:13.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:33:13.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:33:13.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:13.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:33:13.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:13.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:13.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:33:13.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:33:14.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1747 2024-09-18T00:33:14.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:14.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:14.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:33:14.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:14.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:14.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:33:14.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:33:14.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:14.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:33:14.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:14.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:14.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:33:14.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:33:15.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:15.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:15.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:15.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:15.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:15.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:15.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:15.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:15.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:15.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:15.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:15.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1748 2024-09-18T00:33:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1750 2024-09-18T00:33:15.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1749 2024-09-18T00:33:15.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:15.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:16.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:16.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:33:16.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:16.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:16.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:33:16.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:33:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1753 2024-09-18T00:33:16.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:16.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:33:16.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:16.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:16.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:33:16.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:33:16.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1752 2024-09-18T00:33:16.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:16.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1754 2024-09-18T00:33:16.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1751 2024-09-18T00:33:16.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:16.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:16.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:17.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:17.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:33:17.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:17.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:17.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:33:17.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:33:18.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1755 2024-09-18T00:33:18.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1756 2024-09-18T00:33:18.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:18.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:18.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:33:18.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:18.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:18.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:33:18.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:33:18.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:18.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:33:18.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:18.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:18.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:33:18.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:33:18.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:18.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:33:18.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:18.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:18.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:33:18.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:33:19.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1757 2024-09-18T00:33:19.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:19.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:33:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:33:19.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:33:19.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:19.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:19.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:19.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:33:19.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:19.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:19.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:19.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:19.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:19.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:19.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:33:19.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:33:20.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1758 2024-09-18T00:33:20.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:20.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1759 2024-09-18T00:33:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1760 2024-09-18T00:33:20.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:20.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:21.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1761 2024-09-18T00:33:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:33:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:33:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:33:21.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:21.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1764 2024-09-18T00:33:21.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1763 2024-09-18T00:33:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1762 2024-09-18T00:33:21.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:21.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:33:21.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:21.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:21.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:33:21.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:33:21.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:21.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:21.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:22.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:22.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:33:22.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:22.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:22.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:33:22.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:33:23.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:23.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:33:23.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:23.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:23.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:33:23.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:33:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1766 2024-09-18T00:33:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:33:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:33:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:33:23.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1765 2024-09-18T00:33:23.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:23.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:33:23.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:23.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:23.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:33:23.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:33:23.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:24.586 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:33:24.579+0000 7fd4c158b640 -1 reset not still connected to 0x56436bbce270 2024-09-18T00:33:24.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1767 2024-09-18T00:33:24.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:24.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:24.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:24.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:24.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:24.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:24.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:24.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:24.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:33:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:33:25.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1768 2024-09-18T00:33:25.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1769 2024-09-18T00:33:25.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1770 2024-09-18T00:33:25.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:25.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:25.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:26.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1771 2024-09-18T00:33:26.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1773 2024-09-18T00:33:26.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1774 2024-09-18T00:33:26.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:26.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:26.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1772 2024-09-18T00:33:26.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:26.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:26.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:26.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:33:26.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:26.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:26.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:33:26.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:33:26.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:33:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:33:26.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:33:27.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:27.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:33:27.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:27.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:27.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:33:27.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:33:28.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1776 2024-09-18T00:33:28.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1775 2024-09-18T00:33:28.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:33:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:33:28.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:33:28.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:28.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:28.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:33:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:33:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:33:28.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:28.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:33:28.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:33:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:33:29.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:29.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:33:29.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:29.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:29.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:33:29.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:33:29.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:29.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:33:29.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:29.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:29.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:33:29.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:33:29.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:29.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:29.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:29.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:29.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:29.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:29.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1777 2024-09-18T00:33:29.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:29.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:29.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:29.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:29.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:29.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:29.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:30.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1780 2024-09-18T00:33:30.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1779 2024-09-18T00:33:30.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1778 2024-09-18T00:33:30.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:30.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:30.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:31.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1784 2024-09-18T00:33:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:33:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:33:31.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:33:31.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1782 2024-09-18T00:33:31.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:31.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1783 2024-09-18T00:33:31.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:31.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:33:31.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:31.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:31.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:33:31.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:33:31.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1781 2024-09-18T00:33:31.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:31.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:31.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:32.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:32.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:33:32.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:32.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:32.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:33:32.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:33:33.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:33.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:33:33.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:33.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:33.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:33:33.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:33:33.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1785 2024-09-18T00:33:33.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1786 2024-09-18T00:33:33.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:33.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:33:33.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:33.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:33.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:33:33.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:33:33.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:33.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:33:33.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:33.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:33.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:33:33.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:33:33.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:33.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:34.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:34.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:33:34.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:34.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:34.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:33:34.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:33:34.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1787 2024-09-18T00:33:34.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:34.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:34.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:34.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:34.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:34.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:34.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:34.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:34.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:34.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:34.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:34.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:33:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:33:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:33:35.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1790 2024-09-18T00:33:35.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1788 2024-09-18T00:33:35.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:35.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:35.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1789 2024-09-18T00:33:35.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:36.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1794 2024-09-18T00:33:36.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1792 2024-09-18T00:33:36.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:36.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:33:36.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:36.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:36.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:33:36.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:33:36.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1793 2024-09-18T00:33:36.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:36.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:33:36.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:36.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:36.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:33:36.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:33:36.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1791 2024-09-18T00:33:36.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:36.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:36.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:37.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:37.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:33:37.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:37.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:37.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:33:37.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:33:38.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1796 2024-09-18T00:33:38.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1795 2024-09-18T00:33:38.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:38.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:33:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:33:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:33:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:33:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:33:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:33:38.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:38.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:33:38.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:38.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:38.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:33:38.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:33:39.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1797 2024-09-18T00:33:39.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:33:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:33:39.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:33:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:39.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:39.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:39.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:33:39.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:39.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:39.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:33:39.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:33:40.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1798 2024-09-18T00:33:40.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1799 2024-09-18T00:33:40.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:40.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1800 2024-09-18T00:33:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:40.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:41.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1801 2024-09-18T00:33:41.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1803 2024-09-18T00:33:41.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1802 2024-09-18T00:33:41.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:41.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1804 2024-09-18T00:33:41.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:33:41.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:33:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:33:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:33:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:33:41.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:41.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:42.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:33:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:33:42.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:33:43.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:43.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:33:43.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:43.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:43.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:33:43.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:33:43.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1805 2024-09-18T00:33:43.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1806 2024-09-18T00:33:43.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:43.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:33:43.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:43.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:43.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:33:43.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:33:43.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:43.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:33:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:33:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:33:44.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:44.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:44.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:44.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:44.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:44.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:44.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:44.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:44.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:44.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:44.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:44.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:44.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:44.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:33:44.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:44.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:44.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:33:44.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:33:44.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:44.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:33:44.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:44.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:44.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:33:44.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:33:44.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1807 2024-09-18T00:33:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:45.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1810 2024-09-18T00:33:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1809 2024-09-18T00:33:45.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:45.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:45.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1808 2024-09-18T00:33:45.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:46.531 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:33:46.526+0000 7fd4c158b640 -1 reset not still connected to 0x564360789520 2024-09-18T00:33:46.531 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:33:46.526+0000 7fd4c158b640 -1 reset not still connected to 0x56436bbce270 2024-09-18T00:33:46.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1813 2024-09-18T00:33:46.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1812 2024-09-18T00:33:46.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1814 2024-09-18T00:33:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:33:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:33:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:33:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:33:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:33:46.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:33:46.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:46.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:46.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1811 2024-09-18T00:33:46.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:46.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:47.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:47.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:33:47.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:47.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:47.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:33:47.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:33:48.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1815 2024-09-18T00:33:48.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1816 2024-09-18T00:33:48.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:48.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:48.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:48.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:33:48.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:48.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:48.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:33:48.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:33:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:33:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:33:48.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:33:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:33:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:33:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:33:49.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1817 2024-09-18T00:33:49.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:49.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:49.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:33:49.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:49.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:49.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:33:49.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:33:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:49.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:49.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:49.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:33:49.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:49.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:49.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:33:49.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:33:49.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:49.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:49.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:49.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:49.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:49.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:50.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1819 2024-09-18T00:33:50.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:50.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1818 2024-09-18T00:33:50.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1820 2024-09-18T00:33:50.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:50.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:33:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:33:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:33:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:33:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:33:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:33:51.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1821 2024-09-18T00:33:51.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1822 2024-09-18T00:33:51.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1823 2024-09-18T00:33:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1824 2024-09-18T00:33:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:51.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:51.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:33:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:33:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:33:53.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:53.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:33:53.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:53.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:53.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:33:53.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:33:53.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1826 2024-09-18T00:33:53.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1825 2024-09-18T00:33:53.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:53.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:53.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:33:53.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:53.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:53.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:33:53.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:33:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:33:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:33:53.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:33:54.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1827 2024-09-18T00:33:54.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:54.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:54.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:33:54.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:54.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:54.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:33:54.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:33:54.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:54.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:33:54.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:54.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:54.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:33:54.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:33:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:54.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:54.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:54.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:54.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:54.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1829 2024-09-18T00:33:55.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1830 2024-09-18T00:33:55.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1828 2024-09-18T00:33:55.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:55.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:56.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:56.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:33:56.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:56.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:56.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:33:56.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:33:56.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1832 2024-09-18T00:33:56.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1831 2024-09-18T00:33:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:33:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:33:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:33:56.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:56.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:56.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1834 2024-09-18T00:33:56.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1833 2024-09-18T00:33:56.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:56.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:33:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:33:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:33:58.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1835 2024-09-18T00:33:58.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1836 2024-09-18T00:33:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:33:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:33:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:33:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:33:58.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:33:58.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:33:58.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:33:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:33:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:33:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1837 2024-09-18T00:33:59.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:59.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:33:59.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:33:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:33:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:33:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:33:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:33:59.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:33:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:33:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:33:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:33:59.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:33:59.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:33:59.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:33:59.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:33:59.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:33:59.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:34:00.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1839 2024-09-18T00:34:00.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1840 2024-09-18T00:34:00.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1838 2024-09-18T00:34:00.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:00.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:00.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:01.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:01.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:34:01.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:01.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:01.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:34:01.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:34:01.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:01.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:34:01.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:01.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:01.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:34:01.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:34:01.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1842 2024-09-18T00:34:01.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1843 2024-09-18T00:34:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1844 2024-09-18T00:34:01.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1841 2024-09-18T00:34:01.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:02.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:02.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:34:02.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:02.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:02.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:34:02.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:34:03.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1846 2024-09-18T00:34:03.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1845 2024-09-18T00:34:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:34:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:34:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:34:03.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:03.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:03.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:03.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:34:03.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:03.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:03.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:34:03.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:34:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:34:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:34:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:34:04.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1847 2024-09-18T00:34:04.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:04.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:34:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:34:04.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:34:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:34:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:34:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:34:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:34:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:34:04.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:34:04.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:04.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:34:04.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:04.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:04.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:34:04.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:34:05.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1850 2024-09-18T00:34:05.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1848 2024-09-18T00:34:05.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1849 2024-09-18T00:34:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:05.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:06.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1853 2024-09-18T00:34:06.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:06.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:34:06.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:06.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:06.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:34:06.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:34:06.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:34:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:34:06.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:34:06.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1851 2024-09-18T00:34:06.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1854 2024-09-18T00:34:06.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:06.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1852 2024-09-18T00:34:06.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:06.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:34:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:34:07.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:34:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1856 2024-09-18T00:34:08.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1855 2024-09-18T00:34:08.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:08.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:34:08.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:08.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:08.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:34:08.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:34:08.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:08.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:08.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:08.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:34:08.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:08.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:08.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:34:08.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:34:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:34:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:34:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:34:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1857 2024-09-18T00:34:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:09.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:09.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:34:09.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:09.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:09.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:34:09.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:34:09.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:09.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:34:09.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:09.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:09.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:34:09.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:34:09.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:09.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:34:09.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:09.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:09.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:34:09.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:34:09.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:09.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:34:09.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:09.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:09.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:34:09.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:34:10.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1860 2024-09-18T00:34:10.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1858 2024-09-18T00:34:10.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:10.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1859 2024-09-18T00:34:10.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:10.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:11.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:11.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:34:11.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:11.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:11.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:34:11.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:34:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:34:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:34:11.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:34:11.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1864 2024-09-18T00:34:11.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1861 2024-09-18T00:34:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1863 2024-09-18T00:34:11.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1862 2024-09-18T00:34:11.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:11.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:11.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:11.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:12.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:12.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:34:12.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:12.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:12.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:34:12.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:34:13.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1865 2024-09-18T00:34:13.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1866 2024-09-18T00:34:13.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:13.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:13.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:13.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:34:13.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:13.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:13.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:34:13.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:34:13.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:13.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:34:13.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:13.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:13.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:34:13.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:34:13.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:13.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:34:13.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:13.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:13.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:34:13.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:34:14.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1867 2024-09-18T00:34:14.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:14.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:14.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:34:14.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:14.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:14.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:34:14.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:34:14.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:14.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:34:14.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:14.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:14.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:34:14.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:34:14.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:14.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:34:14.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:14.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:14.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:34:14.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:34:14.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:14.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:34:14.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:14.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:14.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:34:14.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:34:15.376 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:34:15.373+0000 7fd4c158b640 -1 reset not still connected to 0x564358707ba0 2024-09-18T00:34:15.377 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:34:15.373+0000 7fd4c158b640 -1 reset not still connected to 0x564358707d40 2024-09-18T00:34:15.377 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:34:15.373+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a340 2024-09-18T00:34:15.377 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:34:15.373+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T00:34:15.377 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:34:15.373+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T00:34:15.377 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:34:15.373+0000 7fd4c158b640 -1 reset not still connected to 0x56435e9100d0 2024-09-18T00:34:15.377 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:34:15.373+0000 7fd4c158b640 -1 reset not still connected to 0x564360789520 2024-09-18T00:34:15.377 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:34:15.373+0000 7fd4c158b640 -1 reset not still connected to 0x56436bbce270 2024-09-18T00:34:15.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1868 2024-09-18T00:34:15.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1870 2024-09-18T00:34:15.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:15.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1869 2024-09-18T00:34:15.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:15.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:16.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:16.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:34:16.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:16.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:16.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:34:16.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:34:16.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:16.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:34:16.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:16.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:16.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:34:16.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:34:16.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1873 2024-09-18T00:34:16.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:16.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1872 2024-09-18T00:34:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:16.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1874 2024-09-18T00:34:16.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1871 2024-09-18T00:34:16.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:16.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:17.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:17.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:34:17.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:17.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:17.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:34:17.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:34:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1876 2024-09-18T00:34:18.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1875 2024-09-18T00:34:18.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:18.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:18.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:18.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:34:18.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:18.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:18.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:34:18.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:34:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:34:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:34:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:34:18.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:18.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:34:18.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:18.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:18.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:34:18.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:34:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1877 2024-09-18T00:34:19.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:19.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:34:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:34:19.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:34:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:34:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:34:19.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:34:19.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:19.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:34:19.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:19.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:19.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:34:19.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:34:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:34:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:34:19.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1878 2024-09-18T00:34:20.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1880 2024-09-18T00:34:20.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:20.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:20.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1879 2024-09-18T00:34:20.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:21.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1881 2024-09-18T00:34:21.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:34:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:34:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:34:21.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:21.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:34:21.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:21.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:21.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:34:21.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:34:21.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1882 2024-09-18T00:34:21.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1883 2024-09-18T00:34:21.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:21.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:21.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1884 2024-09-18T00:34:21.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:22.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:22.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:34:22.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:22.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:22.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:34:22.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:34:23.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1885 2024-09-18T00:34:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:34:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:34:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:34:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:34:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:34:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:34:23.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1886 2024-09-18T00:34:23.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:23.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:23.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:34:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:34:23.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:34:23.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:24.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1887 2024-09-18T00:34:24.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:24.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:34:24.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:24.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:24.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:34:24.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:34:24.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:24.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:24.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:34:24.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:24.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:24.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:34:24.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:34:24.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:24.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:34:24.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:24.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:24.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:34:24.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:34:24.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:24.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:34:24.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:24.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:24.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:34:24.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:34:25.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1890 2024-09-18T00:34:25.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1888 2024-09-18T00:34:25.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:25.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:25.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1889 2024-09-18T00:34:25.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:26.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1892 2024-09-18T00:34:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:34:26.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:26.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:26.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:34:26.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:34:26.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1891 2024-09-18T00:34:26.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:26.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1893 2024-09-18T00:34:26.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:26.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1894 2024-09-18T00:34:26.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:26.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:34:26.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:34:26.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:34:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:27.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:27.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:34:27.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:27.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:27.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:34:27.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:34:28.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1895 2024-09-18T00:34:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:34:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:34:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:34:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:34:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:28.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:34:28.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:34:28.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1896 2024-09-18T00:34:28.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:28.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:28.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:34:28.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:28.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:28.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:34:28.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:34:29.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:29.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:34:29.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:29.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:29.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:34:29.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:34:29.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:29.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:34:29.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:29.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:29.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:34:29.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:34:29.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1897 2024-09-18T00:34:29.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:29.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:29.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:34:29.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:29.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:29.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:34:29.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:34:29.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:29.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:34:29.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:29.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:29.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:34:29.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:34:30.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1899 2024-09-18T00:34:30.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1898 2024-09-18T00:34:30.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1900 2024-09-18T00:34:30.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:30.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:30.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:31.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1902 2024-09-18T00:34:31.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1901 2024-09-18T00:34:31.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:31.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1903 2024-09-18T00:34:31.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:31.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:34:31.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:31.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:31.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:34:31.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:34:31.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:31.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1904 2024-09-18T00:34:31.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:31.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:31.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:34:31.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:31.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:31.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:34:31.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:34:31.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:32.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:32.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:34:32.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:32.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:32.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:34:32.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:34:33.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1906 2024-09-18T00:34:33.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1905 2024-09-18T00:34:33.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:33.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:33.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:33.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:34:33.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:33.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:33.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:34:33.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:34:33.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:33.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:34:33.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:33.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:33.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:34:33.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:34:33.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:33.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:34:33.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:33.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:33.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:34:33.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:34:34.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:34.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:34:34.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:34.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:34.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:34:34.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:34:34.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1907 2024-09-18T00:34:34.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:34:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:34:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:34:34.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:34.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:34:34.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:34.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:34.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:34:34.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:34:34.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:34:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:34:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:34:35.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1910 2024-09-18T00:34:35.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1908 2024-09-18T00:34:35.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:35.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1909 2024-09-18T00:34:35.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:35.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:36.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1911 2024-09-18T00:34:36.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:36.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:34:36.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:36.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:36.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:34:36.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:34:36.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:36.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:34:36.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:36.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:36.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:34:36.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:34:36.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1913 2024-09-18T00:34:36.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1914 2024-09-18T00:34:36.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1912 2024-09-18T00:34:36.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:36.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:36.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:36.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:34:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:34:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:34:38.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:34:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:34:38.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:34:38.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1915 2024-09-18T00:34:38.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1916 2024-09-18T00:34:38.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:38.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:34:38.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:38.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:38.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:34:38.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:34:38.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:38.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:34:38.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:38.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:38.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:34:38.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:34:38.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:38.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:39.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1917 2024-09-18T00:34:39.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:34:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:34:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:34:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:34:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:34:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:34:39.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:39.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:39.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:34:39.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:39.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:39.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:34:39.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:34:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:34:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:34:39.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:34:40.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1920 2024-09-18T00:34:40.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1918 2024-09-18T00:34:40.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1919 2024-09-18T00:34:40.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:41.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1922 2024-09-18T00:34:41.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1924 2024-09-18T00:34:41.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:41.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:34:41.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:41.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:41.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:34:41.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:34:41.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:34:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:34:41.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:34:41.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:41.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1923 2024-09-18T00:34:41.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:41.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1921 2024-09-18T00:34:41.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:41.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:42.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:34:42.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:42.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:42.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:34:42.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:34:43.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1925 2024-09-18T00:34:43.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1926 2024-09-18T00:34:43.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:43.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:43.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:43.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:34:43.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:43.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:43.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:34:43.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:34:43.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:43.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:34:43.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:43.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:43.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:34:43.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:34:43.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:43.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:34:43.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:43.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:43.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:34:43.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:34:44.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1927 2024-09-18T00:34:44.765 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:34:44.762+0000 7f849c00d640 -1 reset not still connected to 0x5629cbaf1930 2024-09-18T00:34:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:44.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:44.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:34:44.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:44.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:44.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:34:44.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:34:44.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:44.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:34:44.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:44.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:44.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:34:44.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:34:44.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:44.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:34:44.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:34:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:34:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:34:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:44.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:44.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:34:44.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:34:45.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1930 2024-09-18T00:34:45.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1928 2024-09-18T00:34:45.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1929 2024-09-18T00:34:45.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:45.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:46.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:46.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:34:46.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:46.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:46.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:34:46.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:34:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:34:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:34:46.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:34:46.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1934 2024-09-18T00:34:46.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1933 2024-09-18T00:34:46.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:46.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1931 2024-09-18T00:34:46.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1932 2024-09-18T00:34:46.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:46.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:46.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:47.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:47.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:34:47.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:47.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:47.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:34:47.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:34:48.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1935 2024-09-18T00:34:48.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1936 2024-09-18T00:34:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:48.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:34:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:34:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:34:48.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:48.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:34:48.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:48.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:48.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:34:48.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:34:48.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:34:49.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1937 2024-09-18T00:34:49.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:34:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:34:49.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:34:49.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:34:49.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:49.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:34:49.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:49.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:49.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:34:49.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:34:49.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:49.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:34:49.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:49.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:49.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:34:49.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:34:50.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1938 2024-09-18T00:34:50.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1939 2024-09-18T00:34:50.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1940 2024-09-18T00:34:50.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:50.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:50.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:50.705 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:34:50.702+0000 7f849c00d640 -1 reset not still connected to 0x5629cbaf1930 2024-09-18T00:34:51.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1941 2024-09-18T00:34:51.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:51.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:34:51.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:51.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:51.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:34:51.645 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:34:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:34:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1943 2024-09-18T00:34:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:34:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:34:51.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1944 2024-09-18T00:34:51.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1942 2024-09-18T00:34:51.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:51.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:51.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:52.622 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:34:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:34:52.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:34:53.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1945 2024-09-18T00:34:53.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:53.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:34:53.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:53.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:53.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:34:53.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:34:53.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1946 2024-09-18T00:34:53.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:53.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:34:53.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:53.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:53.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:34:53.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:34:53.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:53.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:34:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:34:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:34:53.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:53.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:54.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1947 2024-09-18T00:34:54.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:34:54.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 86287 2024-09-18T00:34:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:34:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:34:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:34:54.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:54.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:34:54.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:54.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:54.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:34:54.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:34:54.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:34:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:34:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:34:55.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1948 2024-09-18T00:34:55.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1949 2024-09-18T00:34:55.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1950 2024-09-18T00:34:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:55.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:55.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:56.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1952 2024-09-18T00:34:56.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:56.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:34:56.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:56.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:56.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:34:56.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:34:56.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1953 2024-09-18T00:34:56.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:34:56.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 86460 2024-09-18T00:34:56.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1951 2024-09-18T00:34:56.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:56.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:56.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:57.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:57.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:34:57.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:57.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:57.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:34:57.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:34:58.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1954 2024-09-18T00:34:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:34:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:34:58.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:34:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:34:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:34:58.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:34:58.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:34:58.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 86617 2024-09-18T00:34:58.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:59.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1955 2024-09-18T00:34:59.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:34:59.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:59.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:34:59.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:59.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:59.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:34:59.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:34:59.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:34:59.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:34:59.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:34:59.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:34:59.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:34:59.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:34:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:34:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 86772 2024-09-18T00:35:00.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1956 2024-09-18T00:35:00.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:35:00.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1957 2024-09-18T00:35:00.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:35:01.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1959 2024-09-18T00:35:01.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1958 2024-09-18T00:35:01.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:35:01.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:35:01.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:35:01.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:35:01.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:35:01.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:35:01.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:35:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:35:02.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:35:02.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 86964 2024-09-18T00:35:03.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1960 2024-09-18T00:35:03.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:35:03.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:35:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:35:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:35:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:35:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:35:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:35:03.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:35:03.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:35:03.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:35:03.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:35:03.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:35:03.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:35:04.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:35:04.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:35:04.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:35:04.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:35:04.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:35:04.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:35:04.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:35:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 87139 2024-09-18T00:35:05.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1961 2024-09-18T00:35:05.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:35:05.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1962 2024-09-18T00:35:05.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:35:06.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:35:06.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 87355 2024-09-18T00:35:06.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1963 2024-09-18T00:35:06.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:35:08.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:35:08.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 87558 2024-09-18T00:35:08.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:35:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:35:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:35:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:35:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:35:08.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:35:09.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:35:09.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 87802 2024-09-18T00:35:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1964 2024-09-18T00:35:10.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:35:13.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:35:13.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:35:13.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 86155 2024-09-18T00:37:27.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:37:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:37:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:37:27.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 86288 2024-09-18T00:37:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:37:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:37:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:37:27.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 86461 2024-09-18T00:37:29.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:37:29.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:37:29.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:37:29.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 86618 2024-09-18T00:37:31.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:37:31.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:37:31.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:37:31.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 86773 2024-09-18T00:37:33.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:37:33.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:37:33.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:37:33.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 86965 2024-09-18T00:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:37:35.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 87140 2024-09-18T00:37:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:37:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:37:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:37:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 87356 2024-09-18T00:37:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:37:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:37:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:37:40.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 87559 2024-09-18T00:37:42.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:37:42.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:37:42.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:37:42.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 87803 2024-09-18T00:37:44.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:37:44.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:37:44.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:37:44.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:37:44.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:37:44.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:37:44.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 1 2024-09-18T00:37:44.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:37:44.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 2 2024-09-18T00:37:44.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd0 2024-09-18T00:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary1 2024-09-18T00:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary1 2024-09-18T00:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary1 2024-09-18T00:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd1 2024-09-18T00:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary2 2024-09-18T00:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary2 2024-09-18T00:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary2 2024-09-18T00:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 3 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd2 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary3 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary3 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary3 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 4 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd3 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 5 2024-09-18T00:37:44.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary4 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary4 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary4 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd4 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary5 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary5 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary5 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 6 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd5 2024-09-18T00:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary6 2024-09-18T00:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary6 2024-09-18T00:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary6 2024-09-18T00:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:37:44.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 8 2024-09-18T00:37:44.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd7 2024-09-18T00:37:44.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary8 2024-09-18T00:37:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary8 2024-09-18T00:37:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:37:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:37:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary8 2024-09-18T00:37:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:37:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:37:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:37:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 93593 2024-09-18T00:37:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 10 2024-09-18T00:37:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd9 2024-09-18T00:37:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary10 2024-09-18T00:37:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary10 2024-09-18T00:37:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:37:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary10 2024-09-18T00:37:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 9 2024-09-18T00:37:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd8 2024-09-18T00:37:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary9 2024-09-18T00:37:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary9 2024-09-18T00:37:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:37:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary9 2024-09-18T00:37:44.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 7 2024-09-18T00:37:44.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd6 2024-09-18T00:37:44.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary7 2024-09-18T00:37:44.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary7 2024-09-18T00:37:44.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:37:44.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary7 2024-09-18T00:37:45.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:37:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:37:45.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:37:45.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:37:45.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:37:45.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:37:45.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:37:45.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:37:45.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:37:45.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:37:45.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:37:45.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:37:45.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:37:45.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:37:45.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:37:45.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:37:45.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:37:45.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:37:45.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:37:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:38:41.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=5bed9ee2604f292dbb674bb2360a6272 2024-09-18T00:38:41.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary9 2024-09-18T00:38:43.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary9 2024-09-18T00:38:43.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:43.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:43.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:38:43.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary9 2024-09-18T00:38:44.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=fb6408ac06ae32a1f8bc6e96e17201d1 2024-09-18T00:38:44.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary5 2024-09-18T00:38:45.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:38:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+unknown 2024-09-18T00:38:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:38:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary5 2024-09-18T00:38:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:45.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:45.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:38:45.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary5 2024-09-18T00:38:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T00:38:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:46.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:38:46.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:46.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:46.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:46.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:46.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:46.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:38:46.182 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:38:46.180+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d69179e1a0 2024-09-18T00:38:46.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:38:46.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a3cef58b-69ac-4f11-993d-23070a4b6c51 2024-09-18T00:38:46.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:46.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:46.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:46.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:44 2024-09-18T00:38:46.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:46.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:46.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:46.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:38:46.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:45' 2024-09-18T00:38:46.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:46.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:46.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:46.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:46.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:46.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:46.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:46.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:46.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:46.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:44' 2024-09-18T00:38:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+unknown '2024-09-18 00:38:44' 2024-09-18T00:38:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:38:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:44' 2024-09-18T00:38:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:46.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:46.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:47.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2187264.0,"last_snapshot_bytes":2674688,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619690,"remote_snapshot_timestamp":1726619690,"replay_state":"idle"}2024-09-18 00:38:39image-primary10e4321661-07df-4164-962e-0d882a261febup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2158592.0,"last_snapshot_bytes":2560000,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619708,"remote_snapshot_timestamp":1726619708,"replay_state":"idle"}2024-09-18 00:38:39image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2164736.0,"last_snapshot_bytes":2555904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619692,"remote_snapshot_timestamp":1726619692,"replay_state":"idle"}2024-09-18 00:38:39image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2164736.0,"last_snapshot_bytes":2543616,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619694,"remote_snapshot_timestamp":1726619694,"replay_state":"idle"}2024-09-18 00:38:39image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1708032.0,"last_snapshot_bytes":1601536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619695,"remote_snapshot_timestamp":1726619695,"replay_state":"idle"}2024-09-18 00:38:39image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":182272.0,"last_snapshot_bytes":139264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619698,"remote_snapshot_timestamp":1726619698,"replay_state":"idle"}2024-09-18 00:38:39image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":182272.0,"last_snapshot_bytes":135168,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619700,"remote_snapshot_timestamp":1726619700,"replay_state":"idle"}2024-09-18 00:38:39image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2650112.0,"last_snapshot_bytes":5165056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619702,"remote_snapshot_timestamp":1726619702,"replay_state":"idle"}2024-09-18 00:38:39image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2166784.0,"last_snapshot_bytes":2555904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619704,"remote_snapshot_timestamp":1726619704,"replay_state":"idle"}2024-09-18 00:38:39image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:38:45' 2024-09-18T00:38:47.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:47.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:38:47.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=7ef2ecdb2fc1cc6b6aea55badd6ec32b 2024-09-18T00:38:47.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary2 2024-09-18T00:38:47.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:44' 2024-09-18T00:38:47.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:38:47.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:47.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:47.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:47.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:47.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary9 up+unknown 2024-09-18T00:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:47.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:47.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=a3eb34d8656e77c97a2adaaaa012bf55 2024-09-18T00:38:47.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary7 2024-09-18T00:38:48.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:38:48.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+unknown 2024-09-18T00:38:48.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:48.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:48.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:38:48.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:48.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:48.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:48.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:48.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:48.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=9ff603d2e3407c4c81fe450559f7c34c 2024-09-18T00:38:48.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary4 2024-09-18T00:38:48.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=19f7e4876bc03806f7b69cff43f77c06 2024-09-18T00:38:48.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary1 2024-09-18T00:38:48.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=9129d0e2c334c6b00ce00aa1c9bdcdf0 2024-09-18T00:38:48.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary10 2024-09-18T00:38:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary9 up+unknown '' '' 2024-09-18T00:38:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:38:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:48.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:48.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:48.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary9 2024-09-18T00:38:48.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=c3983c6c08491397c66e6d783db46a93 2024-09-18T00:38:48.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary3 2024-09-18T00:38:48.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary2 2024-09-18T00:38:48.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:48.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:48.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:38:48.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary2 2024-09-18T00:38:49.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:38:49.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a3cef58b-69ac-4f11-993d-23070a4b6c51 2024-09-18T00:38:49.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:49.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:38:49.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:49.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:45 2024-09-18T00:38:49.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:49.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:38:49.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:49.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:49.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:44' 2024-09-18T00:38:49.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:49.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:49.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:38:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:49.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:38:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:49.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:49.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:38:49.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:49.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:49.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:49.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:38:49.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:49.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:49.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:49.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:49.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:49.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:45' 2024-09-18T00:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary9 up+unknown '2024-09-18 00:38:45' 2024-09-18T00:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:49.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:38:49.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:49.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:45' 2024-09-18T00:38:49.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:49.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:49.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:49.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary7 2024-09-18T00:38:49.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:49.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:49.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:38:49.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary7 2024-09-18T00:38:49.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:38:49.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 0dcf2296-c094-48fc-b25c-a438556ec60f 2024-09-18T00:38:49.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:38:49.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:38:49.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:49.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:44 2024-09-18T00:38:49.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:49.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:49.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:49.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:38:49.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:48' 2024-09-18T00:38:49.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:38:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:38:49.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=a3e1530aeb610c14d1d3039e30c8aa89 2024-09-18T00:38:49.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary8 2024-09-18T00:38:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary1 2024-09-18T00:38:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:38:49.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary1 2024-09-18T00:38:49.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary4 2024-09-18T00:38:49.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:49.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:49.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:38:49.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary4 2024-09-18T00:38:49.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING2842614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2187264.0,"last_snapshot_bytes":2674688,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619690,"remote_snapshot_timestamp":1726619690,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:38:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary10e4321661-07df-4164-962e-0d882a261febup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2158592.0,"last_snapshot_bytes":2560000,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619708,"remote_snapshot_timestamp":1726619708,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:38:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2164736.0,"last_snapshot_bytes":2555904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619692,"remote_snapshot_timestamp":1726619692,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:38:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2164736.0,"last_snapshot_bytes":2543616,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619694,"remote_snapshot_timestamp":1726619694,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:38:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1708032.0,"last_snapshot_bytes":1601536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619695,"remote_snapshot_timestamp":1726619695,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:38:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:49image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":182272.0,"last_snapshot_bytes":135168,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619700,"remote_snapshot_timestamp":1726619700,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:38:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2650112.0,"last_snapshot_bytes":5165056,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619702,"remote_snapshot_timestamp":1726619702,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:38:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2166784.0,"last_snapshot_bytes":2555904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619704,"remote_snapshot_timestamp":1726619704,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:38:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:49' 2024-09-18T00:38:49.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:49.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:38:49.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:49' 2024-09-18T00:38:49.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:38:49.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:49.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:49.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:49.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:49.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:49.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:49.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary9 2024-09-18T00:38:49.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:49.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:49.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:38:49.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:38:49.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary9 2024-09-18T00:38:49.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=7bc1eae4f97ad88dd7a23984877ac1c8 2024-09-18T00:38:49.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary6 2024-09-18T00:38:49.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary10 2024-09-18T00:38:49.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:49.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:49.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:38:49.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary10 2024-09-18T00:38:49.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary3 2024-09-18T00:38:49.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:49.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:49.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:38:49.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary3 2024-09-18T00:38:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary8 2024-09-18T00:38:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:38:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary8 2024-09-18T00:38:50.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary6 2024-09-18T00:38:50.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:50.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:50.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:38:50.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary6 2024-09-18T00:38:50.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:38:50.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:38:50.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:38:51.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd0 2024-09-18T00:38:51.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:38:51.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:38:51.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:38:51.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:38:51.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:38:51.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:38:51.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:38:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+unknown 2024-09-18T00:38:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:38:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:51.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+unknown 2024-09-18T00:38:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:38:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:51.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+unknown 2024-09-18T00:38:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:38:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:51.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:51.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:51.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+unknown 2024-09-18T00:38:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:38:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:51.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:51.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+unknown 2024-09-18T00:38:51.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:51.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:51.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:38:51.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:51.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:51.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:51.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:51.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:51.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:38:51.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+unknown 2024-09-18T00:38:51.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:51.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:51.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:38:51.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:51.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:51.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:51.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:51.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:51.203 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:38:51.199+0000 7f849c00d640 -1 reset not still connected to 0x5629c74a5110 2024-09-18T00:38:51.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:38:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:38:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:51.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 0dcf2296-c094-48fc-b25c-a438556ec60f 2024-09-18T00:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:49 2024-09-18T00:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:51.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:51.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:51.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:51.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:49' 2024-09-18T00:38:51.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:51.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:51.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:51.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:51.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:51.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:51.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:51.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:51.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:51.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:49' 2024-09-18T00:38:51.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+unknown '2024-09-18 00:38:49' 2024-09-18T00:38:51.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:51.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:51.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:38:51.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:51.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:49' 2024-09-18T00:38:51.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:51.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:51.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:51.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING2842544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2187264.0,"last_snapshot_bytes":2674688,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619690,"remote_snapshot_timestamp":1726619690,"replay_state":"idle"}2024-09-18 00:38:39image-primary10e4321661-07df-4164-962e-0d882a261febup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2158592.0,"last_snapshot_bytes":2560000,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619708,"remote_snapshot_timestamp":1726619708,"replay_state":"idle"}2024-09-18 00:38:39image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2164736.0,"last_snapshot_bytes":2555904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619692,"remote_snapshot_timestamp":1726619692,"replay_state":"idle"}2024-09-18 00:38:39image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 00:38:51image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1708032.0,"last_snapshot_bytes":1601536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619695,"remote_snapshot_timestamp":1726619695,"replay_state":"idle"}2024-09-18 00:38:39image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:49image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":182272.0,"last_snapshot_bytes":135168,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619700,"remote_snapshot_timestamp":1726619700,"replay_state":"idle"}2024-09-18 00:38:39image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 00:38:51image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:38:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2166784.0,"last_snapshot_bytes":2555904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619704,"remote_snapshot_timestamp":1726619704,"replay_state":"idle"}2024-09-18 00:38:39image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:49' 2024-09-18T00:38:51.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:51.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:38:51.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:49' 2024-09-18T00:38:51.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:38:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:51.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:51.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary5 up+unknown 2024-09-18T00:38:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:38:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:52.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:38:52.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+unknown 2024-09-18T00:38:52.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:52.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:52.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:38:52.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:52.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:52.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:52.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:52.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:52.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:38:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+unknown 2024-09-18T00:38:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:38:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:52.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T00:38:52.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:52.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:52.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:38:52.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:52.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:52.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:52.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:52.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:52.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:38:52.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:38:52.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:52.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:52.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:38:52.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:38:52.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:38:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:38:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:38:52.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:38:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:38:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:52.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:52.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:38:52.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:38:52.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:52.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:52.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:38:52.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:52.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:52.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:52.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:52.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:52.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:38:52.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T00:38:52.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:52.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:52.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:38:52.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:52.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:52.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:52.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:52.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:52.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:38:52.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:38:52.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 732efaab-7893-47c3-85bb-e14c29d2c94f 2024-09-18T00:38:52.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:38:52.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:38:52.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:52.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:49 2024-09-18T00:38:52.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:52.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:52.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:38:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1708032.0,"last_snapshot_bytes":1601536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619695,"remote_snapshot_timestamp":1726619695,"replay_state":"idle"} 2024-09-18T00:38:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:39' 2024-09-18T00:38:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:38:52.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:52.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 70ed3363-f67f-4aab-a4f4-2e9dca8257a3 2024-09-18T00:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:49 2024-09-18T00:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:38:52.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:51' 2024-09-18T00:38:52.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:52.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:38:52.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:52.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:38:52.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:38:52.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f5aaab3c-2eab-452d-b16b-6eb190381b85 2024-09-18T00:38:52.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:38:52.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:38:52.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:52.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:49 2024-09-18T00:38:52.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:52.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:52.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:38:52.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2164736.0,"last_snapshot_bytes":2555904,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619692,"remote_snapshot_timestamp":1726619692,"replay_state":"idle"} 2024-09-18T00:38:52.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:39' 2024-09-18T00:38:52.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:52.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:38:52.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:52.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:38:52.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:38:52.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: d8d7d52c-37d9-4ee7-a7b4-40649be59525 2024-09-18T00:38:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:38:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:38:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:49 2024-09-18T00:38:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:38:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:51' 2024-09-18T00:38:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:52.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:38:52.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:52.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:38:52.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:38:52.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 77cc37a8-1548-46a1-ac79-91a5d3172bb9 2024-09-18T00:38:52.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:38:52.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:38:52.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:52.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:49 2024-09-18T00:38:52.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:52.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:52.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:52.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:38:52.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:51' 2024-09-18T00:38:52.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:52.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:38:52.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:52.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:38:52.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:38:52.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: e4321661-07df-4164-962e-0d882a261feb 2024-09-18T00:38:52.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:38:52.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:38:52.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:52.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:49 2024-09-18T00:38:52.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:52.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:52.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:52.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:38:52.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:51' 2024-09-18T00:38:52.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:38:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:38:52.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary5 up+unknown '' '' 2024-09-18T00:38:52.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:52.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:52.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:38:52.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:52.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:52.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:52.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:52.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:52.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary5 2024-09-18T00:38:52.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:38:52.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 0dcf2296-c094-48fc-b25c-a438556ec60f 2024-09-18T00:38:52.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:52.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:52.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:52.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:49 2024-09-18T00:38:52.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:52.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:38:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:49' 2024-09-18T00:38:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:52.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:52.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:52.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:52.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:52.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:49' 2024-09-18T00:38:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary5 up+unknown '2024-09-18 00:38:49' 2024-09-18T00:38:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:38:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:49' 2024-09-18T00:38:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:52.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:52.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:52.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:38:51cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:38:51cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:38:52cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:38:51cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:38:52cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:49image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:38:52cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:38:51cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:38:52cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:38:49image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:49' 2024-09-18T00:38:52.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:52.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:38:52.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:49' 2024-09-18T00:38:52.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:38:52.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:52.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:52.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:52.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:52.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:52.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:52.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary5 2024-09-18T00:38:52.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:52.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:52.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:38:52.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:38:52.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary5 2024-09-18T00:38:53.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:38:53.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:53.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:53.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:38:53.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:53.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:53.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:53.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:53.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:53.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:38:53.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T00:38:53.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:53.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:38:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:38:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a8d82560-37fd-4dc8-999f-53c47be6dcd9 2024-09-18T00:38:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:38:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:38:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:49 2024-09-18T00:38:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:38:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:52' 2024-09-18T00:38:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:53.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:38:53.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:53.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:38:53.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:38:53.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 7e38e8df-a146-46c4-95b7-7e5c162fb6b5 2024-09-18T00:38:53.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:38:53.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:49 2024-09-18T00:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:52' 2024-09-18T00:38:53.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:53.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:38:53.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:53.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:38:53.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:38:53.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:38:53.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:38:53.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd1 2024-09-18T00:38:53.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:38:53.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:38:54.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:38:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:38:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:54.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:38:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T00:38:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:38:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:54.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:54.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:54.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:38:54.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:38:54.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:54.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:38:54.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:54.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:54.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:54.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:38:54.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:54.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:38:54.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:54.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:54.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:54.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:54.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:38:54.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:38:54.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:54.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:38:54.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:54.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:54.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:54.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:54.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:38:54.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:54.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:38:54.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T00:38:54.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:54.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:38:54.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:54.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:54.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:54.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:54.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:38:54.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:38:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 70ed3363-f67f-4aab-a4f4-2e9dca8257a3 2024-09-18T00:38:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:54.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:54.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:54.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:54.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:54.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:54.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:54.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:54.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:54.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:54.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:54.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: d8d7d52c-37d9-4ee7-a7b4-40649be59525 2024-09-18T00:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:54.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:54.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:54.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:54.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:54.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:54.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:54.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:54.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:54.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:54.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:54.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:38:54.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 732efaab-7893-47c3-85bb-e14c29d2c94f 2024-09-18T00:38:54.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:54.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:54.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:54.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:54.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:54.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:54.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:54.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:54.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:54.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:54.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:54.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:54.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:54.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:54.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:54.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:54.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:54.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:54.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:38:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 77cc37a8-1548-46a1-ac79-91a5d3172bb9 2024-09-18T00:38:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:54.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:54.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:54.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:54.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:38:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:54.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:54.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:54.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:54.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:54.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:54.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:54.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:54.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:38:54.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: e4321661-07df-4164-962e-0d882a261feb 2024-09-18T00:38:54.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:54.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:54.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:54.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:54.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:54.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:54.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:54.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:54.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:54.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:54.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:54.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:54.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:54.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:54.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:54.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:54.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:54.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:54.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:54.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:54.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:54.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:38:54.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f5aaab3c-2eab-452d-b16b-6eb190381b85 2024-09-18T00:38:54.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:54.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:54.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:54.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:54.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:54.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:54.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:54.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:54.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:54.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:54.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:54.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:54.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:54.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:54.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:54.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:54.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:54.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:54.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:54.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:54.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:54.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:54.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:54.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:54.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:38:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:54.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:54.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:54.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54' 2024-09-18T00:38:54.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:54.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:38:54.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:54.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:38:54.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54' 2024-09-18T00:38:54.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:54.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:38:54.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:54.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:54.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:54.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:54.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:54.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:54.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary3 up+unknown 2024-09-18T00:38:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:38:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:54.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:54.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:54.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:38:54.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:54.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:54.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:54.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:54.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54' 2024-09-18T00:38:54.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:54.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:38:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary4 up+unknown 2024-09-18T00:38:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:38:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:54.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:54.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:54.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:38:54.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:54.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:54.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:54.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary1 up+unknown 2024-09-18T00:38:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:38:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:54.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:54.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54' 2024-09-18T00:38:54.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:54.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:38:54.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:54.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:38:54.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:54.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:54.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:54.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:54.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:54.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:54.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary7 up+unknown 2024-09-18T00:38:54.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:54.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:38:54.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:54.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:54.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:54.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:54.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54' 2024-09-18T00:38:54.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:54.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:38:54.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:54.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54' 2024-09-18T00:38:54.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:54.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:38:54.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:54.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:54.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:54.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:38:54.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:54.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:54.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:54.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:54.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary2 up+unknown 2024-09-18T00:38:54.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:54.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:38:54.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:54.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:54.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:54.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:54.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:54.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:54.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:54.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:54.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:54.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:54.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary10 up+unknown 2024-09-18T00:38:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:38:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:55.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:38:55.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:55.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:38:55.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:55.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:55.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:38:55.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T00:38:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:38:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:38:55.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:38:55.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 7e38e8df-a146-46c4-95b7-7e5c162fb6b5 2024-09-18T00:38:55.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:55.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:55.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:55.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:55.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:55.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:55.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:55.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:55.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:55.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:55.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:55.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:55.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:55.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:55.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:55.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:55.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:55.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:55.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:55.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:55.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:38:55.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:55.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:38:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a8d82560-37fd-4dc8-999f-53c47be6dcd9 2024-09-18T00:38:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:38:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:55.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:55.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:55.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:55.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:55.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:55.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:55.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:55.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:55.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:55.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54' 2024-09-18T00:38:55.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:55.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:38:55.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:55.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:38:55.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:55.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:55.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:55.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:55.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:55.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:55.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary6 up+unknown 2024-09-18T00:38:55.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:38:55.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:55.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:55.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:55.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:55.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:38:54cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:38:54' 2024-09-18T00:38:55.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:55.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:38:55.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:55.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:38:55.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:55.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:55.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:55.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:55.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:55.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:55.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary8 up+unknown 2024-09-18T00:38:55.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:38:55.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:55.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:55.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:38:55.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:38:55.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary3 up+unknown '' '' 2024-09-18T00:38:55.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:38:55.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:55.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:55.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary3 2024-09-18T00:38:55.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary4 up+unknown '' '' 2024-09-18T00:38:55.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:38:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary4 2024-09-18T00:38:55.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary1 up+unknown '' '' 2024-09-18T00:38:55.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:38:55.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:55.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:55.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary1 2024-09-18T00:38:55.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary7 up+unknown '' '' 2024-09-18T00:38:55.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:38:55.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:55.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:55.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary7 2024-09-18T00:38:55.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary2 up+unknown '' '' 2024-09-18T00:38:55.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:38:55.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:55.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:55.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary2 2024-09-18T00:38:55.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary10 up+unknown '' '' 2024-09-18T00:38:55.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:38:55.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary10 2024-09-18T00:38:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:38:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 70ed3363-f67f-4aab-a4f4-2e9dca8257a3 2024-09-18T00:38:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:38:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:55.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:55.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:55.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:55.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:55.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:55.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:55.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:55.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:55.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:55.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary3 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:38:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 732efaab-7893-47c3-85bb-e14c29d2c94f 2024-09-18T00:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:55.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:55.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:55.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:55.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:55.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:55.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:55.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:55.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:55.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:55.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary4 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:55.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:38:55.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:55.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: d8d7d52c-37d9-4ee7-a7b4-40649be59525 2024-09-18T00:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:55.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:55.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:55.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:55.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:55.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:38:55.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f5aaab3c-2eab-452d-b16b-6eb190381b85 2024-09-18T00:38:55.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:55.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:55.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:55.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:38:55.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:55.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:55.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:55.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:55.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:55.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:55.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:55.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:55.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:55.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:55.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:55.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:55.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:55.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:55.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:55.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:55.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:55.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:55.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:55.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:55.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary7 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:38:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 77cc37a8-1548-46a1-ac79-91a5d3172bb9 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:55.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:55.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary2 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: e4321661-07df-4164-962e-0d882a261feb 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:55.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:55.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:55.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:38:55.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:55.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:55.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:55.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:55.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:55.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:55.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:55.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:55.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:55.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:55.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:55.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:55.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:55.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:55.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:55.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:55.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:55.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:55.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:55.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:55.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:55.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:55.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:55.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:55.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:55.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:55.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:55.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary10 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:55.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:38:55.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:55.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary1 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:38:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:55.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:55.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:55.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:56.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}2024-09-18 00:38:54' 2024-09-18T00:38:56.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:56.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:38:56.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:56.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:38:56.084 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:38:56.082+0000 7f52d7c6c640 -1 reset not still connected to 0x55785ba5cc30 2024-09-18T00:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:56.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:56.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:56.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:56.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:56.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:56.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary3 2024-09-18T00:38:56.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:56.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:56.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:38:56.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:38:56.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary3 2024-09-18T00:38:56.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}2024-09-18 00:38:54' 2024-09-18T00:38:56.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:56.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:38:56.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:56.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:38:56.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:56.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:56.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary4 2024-09-18T00:38:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:56.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:38:56.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary4 2024-09-18T00:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}2024-09-18 00:38:54' 2024-09-18T00:38:56.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:56.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:38:56.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}2024-09-18 00:38:54' 2024-09-18T00:38:56.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:56.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:38:56.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:38:56.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:56.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:38:56.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:56.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:56.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}2024-09-18 00:38:54' 2024-09-18T00:38:56.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:56.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:38:56.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:56.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary10 2024-09-18T00:38:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:38:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:38:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary10 2024-09-18T00:38:56.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:56.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:56.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:38:56.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:56.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:56.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:56.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary7 2024-09-18T00:38:56.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:56.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:56.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:38:56.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:38:56.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary7 2024-09-18T00:38:56.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:56.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:56.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:56.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:56.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:56.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:56.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary1 2024-09-18T00:38:56.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:56.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:56.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:38:56.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:38:56.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary1 2024-09-18T00:38:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}2024-09-18 00:38:54' 2024-09-18T00:38:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:38:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:56.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:38:56.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:56.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:56.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:56.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:56.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:56.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:56.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary2 2024-09-18T00:38:56.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:56.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:56.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:38:56.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:38:56.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary2 2024-09-18T00:38:56.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:38:56.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:38:56.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:38:56.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:38:56.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:38:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:38:56.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:38:56.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:38:56.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary6 up+unknown '' '' 2024-09-18T00:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary6 2024-09-18T00:38:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary8 up+unknown '' '' 2024-09-18T00:38:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:38:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:38:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:38:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary8 2024-09-18T00:38:56.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd2 2024-09-18T00:38:56.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:38:56.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:38:56.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd3 2024-09-18T00:38:56.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:38:56.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:38:56.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:38:56.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a8d82560-37fd-4dc8-999f-53c47be6dcd9 2024-09-18T00:38:56.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:56.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:56.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:56.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:56.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:56.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:38:56.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:56.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:56.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:56.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:56.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:38:56.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 7e38e8df-a146-46c4-95b7-7e5c162fb6b5 2024-09-18T00:38:56.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:56.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:56.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:38:56.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54 2024-09-18T00:38:56.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:38:56.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:38:56.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:38:56.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:38:56.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:38:54' 2024-09-18T00:38:56.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:38:56.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:56.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:56.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:38:56.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:56.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:38:56.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:56.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:56.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:38:56.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:38:56.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:56.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:38:56.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:56.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:56.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:56.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:56.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:56.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:56.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:56.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:38:56.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:38:56.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:38:56.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:38:56.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:56.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary8 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:56.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:56.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:56.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:38:56.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:56.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:56.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:56.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:56.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:56.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:56.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary6 up+unknown '2024-09-18 00:38:54' 2024-09-18T00:38:56.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:56.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:56.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:38:56.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:38:56.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:38:54' 2024-09-18T00:38:56.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:38:56.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:38:56.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:38:56.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd4 2024-09-18T00:38:56.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:38:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:38:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}2024-09-18 00:38:54' 2024-09-18T00:38:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:57.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:38:57.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:57.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:38:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary8 2024-09-18T00:38:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:38:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:38:57.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary8 2024-09-18T00:38:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary10e4321661-07df-4164-962e-0d882a261febup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}2024-09-18 00:38:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:38:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:38:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}2024-09-18 00:38:54' 2024-09-18T00:38:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:38:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:38:57.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:38:54' 2024-09-18T00:38:57.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:38:57.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:38:57.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:38:57.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:38:57.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:38:57.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:57.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:38:57.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary6 2024-09-18T00:38:57.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:38:57.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:38:57.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:38:57.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:38:57.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary6 2024-09-18T00:38:57.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:38:57.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:38:57.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:38:57.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:38:57.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:38:57.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:38:57.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:38:57.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:38:57.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:38:57.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:38:57.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:38:57.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:38:57.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:38:57.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:38:57.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:38:57.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd5 2024-09-18T00:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:38:57.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:38:57.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd6 2024-09-18T00:38:57.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:38:57.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:38:57.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd7 2024-09-18T00:38:57.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:38:57.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:38:57.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd8 2024-09-18T00:38:57.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:38:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd9 2024-09-18T00:38:58.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:38:58.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:38:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:40:27.084 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-18T00:40:27.080+0000 7fbbe8b19640 -1 reset not still connected to 0x561a17a220d0 2024-09-18T00:40:32.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=5bed9ee2604f292dbb674bb2360a6272 2024-09-18T00:40:32.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd0 2024-09-18T00:40:34.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 5bed9ee2604f292dbb674bb2360a6272 != \5\b\e\d\9\e\e\2\6\0\4\f\2\9\2\d\b\b\6\7\4\b\b\2\3\6\0\a\6\2\7\2 ]] 2024-09-18T00:40:38.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=fb6408ac06ae32a1f8bc6e96e17201d1 2024-09-18T00:40:38.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd1 2024-09-18T00:40:39.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ fb6408ac06ae32a1f8bc6e96e17201d1 != \f\b\6\4\0\8\a\c\0\6\a\e\3\2\a\1\f\8\b\c\6\e\9\6\e\1\7\2\0\1\d\1 ]] 2024-09-18T00:40:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=19f7e4876bc03806f7b69cff43f77c06 2024-09-18T00:40:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd2 2024-09-18T00:40:46.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 19f7e4876bc03806f7b69cff43f77c06 != \1\9\f\7\e\4\8\7\6\b\c\0\3\8\0\6\f\7\b\6\9\c\f\f\4\3\f\7\7\c\0\6 ]] 2024-09-18T00:40:46.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:40:46.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 93594 2024-09-18T00:40:46.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=9129d0e2c334c6b00ce00aa1c9bdcdf0 2024-09-18T00:40:46.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd3 2024-09-18T00:40:46.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=c3983c6c08491397c66e6d783db46a93 2024-09-18T00:40:46.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd9 2024-09-18T00:40:47.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=a3eb34d8656e77c97a2adaaaa012bf55 2024-09-18T00:40:47.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd6 2024-09-18T00:40:47.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=7ef2ecdb2fc1cc6b6aea55badd6ec32b 2024-09-18T00:40:47.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd5 2024-09-18T00:40:47.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 9129d0e2c334c6b00ce00aa1c9bdcdf0 != \9\1\2\9\d\0\e\2\c\3\3\4\c\6\b\0\0\c\e\0\0\a\a\1\c\9\b\d\c\d\f\0 ]] 2024-09-18T00:40:47.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=a3e1530aeb610c14d1d3039e30c8aa89 2024-09-18T00:40:47.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd8 2024-09-18T00:40:47.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ c3983c6c08491397c66e6d783db46a93 != \c\3\9\8\3\c\6\c\0\8\4\9\1\3\9\7\c\6\6\e\6\d\7\8\3\d\b\4\6\a\9\3 ]] 2024-09-18T00:40:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=9ff603d2e3407c4c81fe450559f7c34c 2024-09-18T00:40:47.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd4 2024-09-18T00:40:48.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=7bc1eae4f97ad88dd7a23984877ac1c8 2024-09-18T00:40:48.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd7 2024-09-18T00:40:48.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ a3eb34d8656e77c97a2adaaaa012bf55 != \a\3\e\b\3\4\d\8\6\5\6\e\7\7\c\9\7\a\2\a\d\a\a\a\a\0\1\2\b\f\5\5 ]] 2024-09-18T00:40:48.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 7ef2ecdb2fc1cc6b6aea55badd6ec32b != \7\e\f\2\e\c\d\b\2\f\c\1\c\c\6\b\6\a\e\a\5\5\b\a\d\d\6\e\c\3\2\b ]] 2024-09-18T00:40:48.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:40:48.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 93595 2024-09-18T00:40:48.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:40:48.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 93598 2024-09-18T00:40:48.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ a3e1530aeb610c14d1d3039e30c8aa89 != \a\3\e\1\5\3\0\a\e\b\6\1\0\c\1\4\d\1\d\3\0\3\9\e\3\0\c\8\a\a\8\9 ]] 2024-09-18T00:40:48.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 9ff603d2e3407c4c81fe450559f7c34c != \9\f\f\6\0\3\d\2\e\3\4\0\7\c\4\c\8\1\f\e\4\5\0\5\5\9\f\7\c\3\4\c ]] 2024-09-18T00:40:48.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:40:48.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 93600 2024-09-18T00:40:48.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:40:48.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 93601 2024-09-18T00:40:48.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 7bc1eae4f97ad88dd7a23984877ac1c8 != \7\b\c\1\e\a\e\4\f\9\7\a\d\8\8\d\d\7\a\2\3\9\8\4\8\7\7\a\c\1\c\8 ]] 2024-09-18T00:40:48.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 93604 2024-09-18T00:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 93606 2024-09-18T00:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 93607 2024-09-18T00:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 93609 2024-09-18T00:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:40:48.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:40:48.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary1 2024-09-18T00:40:48.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:48.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:48.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:40:48.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:40:48.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+replaying 2024-09-18T00:40:48.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:48.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:48.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:40:48.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:40:48.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:40:48.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:40:48.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:40:48.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:40:49.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+replaying '' '' 2024-09-18T00:40:49.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:49.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:49.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:40:49.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:40:49.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:40:49.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:40:49.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:40:49.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:40:49.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:40:49.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:40:49.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 77cc37a8-1548-46a1-ac79-91a5d3172bb9 2024-09-18T00:40:49.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:40:49.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70590464.0,"last_snapshot_bytes":70590464,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"} 2024-09-18T00:40:49.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:40:49.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:40:39 2024-09-18T00:40:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:40:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:40:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:40:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:40:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:40:49' 2024-09-18T00:40:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:40:50.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:40:50.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:40:50.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70590464.0,"last_snapshot_bytes":70590464,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"} 2024-09-18T00:40:50.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:40:50.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:40:50.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:40:50.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:40:50.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:40:50.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:40:50.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:40:50.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:40:50.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:40:50.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+replaying '2024-09-18 00:40:39' 2024-09-18T00:40:50.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:50.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:50.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:40:50.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:40:50.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:40:39' 2024-09-18T00:40:50.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:40:50.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:40:50.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:40:50.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary177cc37a8-1548-46a1-ac79-91a5d3172bb9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70590464.0,"last_snapshot_bytes":70590464,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary10e4321661-07df-4164-962e-0d882a261febup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70250496.0,"last_snapshot_bytes":70250496,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70246400.0,"last_snapshot_bytes":70246400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70262784.0,"last_snapshot_bytes":70262784,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72085504.0,"last_snapshot_bytes":72085504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64143360.0,"last_snapshot_bytes":64143360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619933,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63791104.0,"last_snapshot_bytes":63791104,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60452864.0,"last_snapshot_bytes":60452864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62033920.0,"last_snapshot_bytes":62033920,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619930,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49' 2024-09-18T00:40:50.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:40:50.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:40:50.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:40:50.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:40:50.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:40:50.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:40:50.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:40:50.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:40:50.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:40:50.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:40:50.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary1 2024-09-18T00:40:50.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:40:50.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:50.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:40:50.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary1 2024-09-18T00:40:50.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary1 2024-09-18T00:40:52.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:40:52.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary1 2024-09-18T00:40:52.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:52.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:52.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:40:52.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:40:52.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:40:52.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary1 2024-09-18T00:40:52.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:40:52.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary1 2024-09-18T00:40:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:40:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:40:52.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary2 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+replaying 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:40:52.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:40:53.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+replaying '' '' 2024-09-18T00:40:53.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:53.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:53.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:40:53.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:40:53.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:40:53.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:40:53.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:40:53.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:40:53.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:40:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:40:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f5aaab3c-2eab-452d-b16b-6eb190381b85 2024-09-18T00:40:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:40:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70246400.0,"last_snapshot_bytes":70246400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"} 2024-09-18T00:40:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:40:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:40:39 2024-09-18T00:40:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:40:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:40:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:40:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:40:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:40:49' 2024-09-18T00:40:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:40:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:40:53.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:40:53.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70246400.0,"last_snapshot_bytes":70246400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"} 2024-09-18T00:40:53.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:40:53.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:40:53.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:40:53.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:40:53.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:40:53.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:40:53.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:40:53.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:40:53.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:40:53.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+replaying '2024-09-18 00:40:39' 2024-09-18T00:40:53.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:53.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:53.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:40:53.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:40:53.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:40:39' 2024-09-18T00:40:53.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:40:53.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:40:53.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:40:53.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10e4321661-07df-4164-962e-0d882a261febup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70250496.0,"last_snapshot_bytes":70250496,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary2f5aaab3c-2eab-452d-b16b-6eb190381b85up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70246400.0,"last_snapshot_bytes":70246400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70262784.0,"last_snapshot_bytes":70262784,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72085504.0,"last_snapshot_bytes":72085504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64143360.0,"last_snapshot_bytes":64143360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619933,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63791104.0,"last_snapshot_bytes":63791104,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60452864.0,"last_snapshot_bytes":60452864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62033920.0,"last_snapshot_bytes":62033920,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619930,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:49' 2024-09-18T00:40:53.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:40:53.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:40:53.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:40:53.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:40:53.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:40:53.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:40:53.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:40:53.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:40:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:40:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:40:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary2 2024-09-18T00:40:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:40:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:53.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:40:53.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary2 2024-09-18T00:40:53.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary2 2024-09-18T00:40:55.214 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:40:55.212+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:40:55.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:40:55.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary2 2024-09-18T00:40:55.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:55.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:55.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:40:55.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:40:55.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:40:55.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary2 2024-09-18T00:40:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:40:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary2 2024-09-18T00:40:55.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:40:55.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:40:55.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary3 2024-09-18T00:40:55.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:55.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:55.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:40:55.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:40:55.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+replaying 2024-09-18T00:40:55.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:55.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:55.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:40:55.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:40:55.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:40:55.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:40:55.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:40:55.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:40:55.870 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:55.868+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef1c70 2024-09-18T00:40:55.876 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:55.872+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4000 2024-09-18T00:40:55.876 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:55.872+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4750 2024-09-18T00:40:55.877 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:55.872+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6734b1ee0 2024-09-18T00:40:55.877 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:55.872+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735afee0 2024-09-18T00:40:55.877 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:55.872+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6a2dc1110 2024-09-18T00:40:55.877 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:55.872+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6a9188f70 2024-09-18T00:40:55.877 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:55.872+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6a9189860 2024-09-18T00:40:55.882 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:40:55.880+0000 7f849c00d640 -1 reset not still connected to 0x5629a85e6f70 2024-09-18T00:40:55.885 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:40:55.880+0000 7f849c00d640 -1 reset not still connected to 0x5629a85e7040 2024-09-18T00:40:55.885 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:40:55.880+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:40:55.885 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:40:55.880+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:40:55.885 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:40:55.880+0000 7f849c00d640 -1 reset not still connected to 0x5629cbc1e270 2024-09-18T00:40:55.892 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:40:55.888+0000 7f849c00d640 -1 reset not still connected to 0x5629e215cea0 2024-09-18T00:40:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+replaying '' '' 2024-09-18T00:40:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:40:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:40:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:40:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:40:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:40:56.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:40:56.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:40:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:40:56.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 70ed3363-f67f-4aab-a4f4-2e9dca8257a3 2024-09-18T00:40:56.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:40:56.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70262784.0,"last_snapshot_bytes":70262784,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"} 2024-09-18T00:40:56.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:40:56.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:40:39 2024-09-18T00:40:56.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:40:56.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:40:56.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:40:56.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:40:56.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:40:54' 2024-09-18T00:40:56.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:40:56.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:40:56.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:40:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70262784.0,"last_snapshot_bytes":70262784,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"} 2024-09-18T00:40:56.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:40:56.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:40:56.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:40:56.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:40:56.585 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:40:56.586 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:40:56.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:40:56.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:40:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:40:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+replaying '2024-09-18 00:40:39' 2024-09-18T00:40:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:40:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:40:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:40:39' 2024-09-18T00:40:56.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:40:56.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:40:56.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:40:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK842544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10e4321661-07df-4164-962e-0d882a261febup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70250496.0,"last_snapshot_bytes":70250496,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:54image-primary370ed3363-f67f-4aab-a4f4-2e9dca8257a3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70262784.0,"last_snapshot_bytes":70262784,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:54image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72085504.0,"last_snapshot_bytes":72085504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:54image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64143360.0,"last_snapshot_bytes":64143360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619933,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:54image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63791104.0,"last_snapshot_bytes":63791104,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:54image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60452864.0,"last_snapshot_bytes":60452864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:54image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:54image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62033920.0,"last_snapshot_bytes":62033920,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619930,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:54' 2024-09-18T00:40:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:40:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:40:56.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:40:56.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:40:56.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:40:56.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:40:56.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:40:56.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:40:56.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:40:56.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:40:56.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary3 2024-09-18T00:40:56.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:40:56.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:56.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:40:56.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary3 2024-09-18T00:40:56.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary3 2024-09-18T00:40:58.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:40:58.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary3 2024-09-18T00:40:58.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:58.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:58.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:40:58.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:40:58.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:40:58.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary3 2024-09-18T00:40:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:40:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary3 2024-09-18T00:40:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:40:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:40:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary4 2024-09-18T00:40:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:40:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:40:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+replaying 2024-09-18T00:40:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:40:58.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:40:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:40:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:40:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:40:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:40:59.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+replaying '' '' 2024-09-18T00:40:59.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:59.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:59.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:40:59.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:40:59.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:40:59.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:40:59.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:40:59.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:40:59.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:40:59.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:40:59.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 732efaab-7893-47c3-85bb-e14c29d2c94f 2024-09-18T00:40:59.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:40:59.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72085504.0,"last_snapshot_bytes":72085504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"} 2024-09-18T00:40:59.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:40:59.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:40:39 2024-09-18T00:40:59.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:40:59.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:40:59.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:40:59.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:40:59.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:40:59' 2024-09-18T00:40:59.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:40:59.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:40:59.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:40:59.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72085504.0,"last_snapshot_bytes":72085504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"} 2024-09-18T00:40:59.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:40:59.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:40:59.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:40:59.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:40:59.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:40:59.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:40:59.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:40:59.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:40:59.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:40:59.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+replaying '2024-09-18 00:40:39' 2024-09-18T00:40:59.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:40:59.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:59.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:40:59.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:40:59.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:40:39' 2024-09-18T00:40:59.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:40:59.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:40:59.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:40:59.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10e4321661-07df-4164-962e-0d882a261febup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70250496.0,"last_snapshot_bytes":70250496,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59image-primary4732efaab-7893-47c3-85bb-e14c29d2c94fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72085504.0,"last_snapshot_bytes":72085504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64143360.0,"last_snapshot_bytes":64143360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619933,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63791104.0,"last_snapshot_bytes":63791104,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60452864.0,"last_snapshot_bytes":60452864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62033920.0,"last_snapshot_bytes":62033920,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619930,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59' 2024-09-18T00:40:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:40:59.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:40:59.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:40:59.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:40:59.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:40:59.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:40:59.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:40:59.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:40:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:40:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:40:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary4 2024-09-18T00:40:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:40:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:40:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:40:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary4 2024-09-18T00:40:59.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary4 2024-09-18T00:40:59.975 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:59.964+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4750 2024-09-18T00:40:59.976 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:59.968+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6734b1ee0 2024-09-18T00:40:59.976 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:59.968+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735afee0 2024-09-18T00:40:59.976 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:59.968+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6a2dc1110 2024-09-18T00:40:59.977 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:59.972+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6a9188f70 2024-09-18T00:40:59.977 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:40:59.972+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6a9189860 2024-09-18T00:41:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:41:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary4 2024-09-18T00:41:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:41:00.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:41:00.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary4 2024-09-18T00:41:00.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:00.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary4 2024-09-18T00:41:00.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:00.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:41:00.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary5 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+replaying 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:41:01.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+replaying '' '' 2024-09-18T00:41:01.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:01.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:01.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:01.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:01.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:41:01.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:41:01.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:41:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:41:01.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:41:02.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:41:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 0dcf2296-c094-48fc-b25c-a438556ec60f 2024-09-18T00:41:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:41:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64143360.0,"last_snapshot_bytes":64143360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619933,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"} 2024-09-18T00:41:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:41:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:40:39 2024-09-18T00:41:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:41:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:41:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:41:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:41:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:40:59' 2024-09-18T00:41:02.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:41:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:41:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:41:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64143360.0,"last_snapshot_bytes":64143360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619933,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"} 2024-09-18T00:41:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:41:02.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:41:02.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:41:02.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:41:02.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:41:02.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:41:02.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:41:02.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:41:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:41:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+replaying '2024-09-18 00:40:39' 2024-09-18T00:41:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:40:39' 2024-09-18T00:41:02.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:41:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:41:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:41:02.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK642544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10e4321661-07df-4164-962e-0d882a261febup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70250496.0,"last_snapshot_bytes":70250496,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59image-primary50dcf2296-c094-48fc-b25c-a438556ec60fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64143360.0,"last_snapshot_bytes":64143360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619933,"remote_snapshot_timestamp":1726619933,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63791104.0,"last_snapshot_bytes":63791104,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60452864.0,"last_snapshot_bytes":60452864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62033920.0,"last_snapshot_bytes":62033920,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619930,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:40:59' 2024-09-18T00:41:02.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:41:02.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:41:02.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:41:02.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:41:02.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:41:02.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:41:02.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:41:02.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:41:02.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:02.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:02.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary5 2024-09-18T00:41:02.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:41:02.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:02.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:02.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary5 2024-09-18T00:41:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary5 2024-09-18T00:41:04.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:41:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary5 2024-09-18T00:41:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:41:04.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary5 2024-09-18T00:41:04.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:04.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary5 2024-09-18T00:41:04.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:41:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary6 2024-09-18T00:41:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:41:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+replaying 2024-09-18T00:41:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:04.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:04.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:41:04.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:41:04.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:04.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:41:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+replaying '' '' 2024-09-18T00:41:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:41:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:41:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:41:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:41:05.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:41:05.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:41:05.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 7e38e8df-a146-46c4-95b7-7e5c162fb6b5 2024-09-18T00:41:05.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:41:05.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63791104.0,"last_snapshot_bytes":63791104,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"} 2024-09-18T00:41:05.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:41:05.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:40:39 2024-09-18T00:41:05.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:41:05.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:41:05.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:41:05.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:41:05.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:41:04' 2024-09-18T00:41:05.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:41:05.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:41:05.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:41:05.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63791104.0,"last_snapshot_bytes":63791104,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"} 2024-09-18T00:41:05.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:41:05.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:41:05.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:41:05.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:41:05.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:41:05.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:41:05.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:41:05.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:41:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:41:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+replaying '2024-09-18 00:40:39' 2024-09-18T00:41:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:05.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:40:39' 2024-09-18T00:41:05.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:41:05.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:41:05.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:41:05.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10e4321661-07df-4164-962e-0d882a261febup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70250496.0,"last_snapshot_bytes":70250496,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:04image-primary67e38e8df-a146-46c4-95b7-7e5c162fb6b5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63791104.0,"last_snapshot_bytes":63791104,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:04image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60452864.0,"last_snapshot_bytes":60452864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:04image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:04image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62033920.0,"last_snapshot_bytes":62033920,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619930,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:40:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:04' 2024-09-18T00:41:05.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:41:05.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:41:05.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:41:05.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:41:05.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:41:05.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:41:05.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:41:05.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:41:05.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:05.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:05.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary6 2024-09-18T00:41:05.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:41:05.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:05.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:05.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary6 2024-09-18T00:41:06.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary6 2024-09-18T00:41:07.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:41:07.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary6 2024-09-18T00:41:07.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:07.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:07.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:07.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:07.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:41:07.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary6 2024-09-18T00:41:07.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:07.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary6 2024-09-18T00:41:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:41:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary7 2024-09-18T00:41:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:41:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+replaying 2024-09-18T00:41:07.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:07.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:07.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:07.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:07.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:41:07.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:41:07.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:07.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:41:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+replaying '' '' 2024-09-18T00:41:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:41:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:41:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:41:08.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:41:08.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:41:08.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:41:08.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: d8d7d52c-37d9-4ee7-a7b4-40649be59525 2024-09-18T00:41:08.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:41:08.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60452864.0,"last_snapshot_bytes":60452864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"} 2024-09-18T00:41:08.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:41:08.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:40:39 2024-09-18T00:41:08.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:41:08.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:41:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:41:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:41:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:41:04' 2024-09-18T00:41:08.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:41:08.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:41:08.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:41:08.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60452864.0,"last_snapshot_bytes":60452864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"} 2024-09-18T00:41:08.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:41:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:41:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:41:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:41:08.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:41:08.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:41:08.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:41:08.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:41:08.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:40:39' 2024-09-18T00:41:08.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+replaying '2024-09-18 00:40:39' 2024-09-18T00:41:08.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:08.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:08.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:08.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:08.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:40:39' 2024-09-18T00:41:08.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:41:08.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:41:08.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:41:09.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10e4321661-07df-4164-962e-0d882a261febup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70250496.0,"last_snapshot_bytes":70250496,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:41:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:04image-primary7d8d7d52c-37d9-4ee7-a7b4-40649be59525up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60452864.0,"last_snapshot_bytes":60452864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:41:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:04image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:41:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:09image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62033920.0,"last_snapshot_bytes":62033920,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619930,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:41:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:09' 2024-09-18T00:41:09.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:41:09.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:41:09.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:41:09' 2024-09-18T00:41:09.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:41:09.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:41:09.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:41:09.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:41:09.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:41:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary7 2024-09-18T00:41:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:41:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary7 2024-09-18T00:41:09.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary7 2024-09-18T00:41:11.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:41:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary7 2024-09-18T00:41:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:11.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:41:11.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary7 2024-09-18T00:41:11.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:11.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary7 2024-09-18T00:41:11.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:11.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary8 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+replaying 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:41:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:41:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+replaying '' '' 2024-09-18T00:41:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:41:12.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:41:12.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:41:12.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:41:12.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:41:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:41:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a8d82560-37fd-4dc8-999f-53c47be6dcd9 2024-09-18T00:41:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:41:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"} 2024-09-18T00:41:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:41:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:41:09 2024-09-18T00:41:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:41:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:41:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:41:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:41:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:41:09' 2024-09-18T00:41:12.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:41:12.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:41:12.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:41:12.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"} 2024-09-18T00:41:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:41:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:41:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:41:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:41:12.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:41:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:41:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:41:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:41:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:41:09' 2024-09-18T00:41:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+replaying '2024-09-18 00:41:09' 2024-09-18T00:41:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:41:09' 2024-09-18T00:41:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:41:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:41:12.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:41:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK342544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10e4321661-07df-4164-962e-0d882a261febup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70250496.0,"last_snapshot_bytes":70250496,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:41:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:09image-primary8a8d82560-37fd-4dc8-999f-53c47be6dcd9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70209536.0,"last_snapshot_bytes":70209536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619937,"remote_snapshot_timestamp":1726619937,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:41:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:09image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62033920.0,"last_snapshot_bytes":62033920,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619930,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:41:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:09' 2024-09-18T00:41:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:41:12.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:41:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:41:09' 2024-09-18T00:41:12.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:41:12.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:41:12.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:41:12.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:41:12.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:41:12.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:12.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:12.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary8 2024-09-18T00:41:12.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:41:12.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:12.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:12.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary8 2024-09-18T00:41:12.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary8 2024-09-18T00:41:14.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:41:14.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary8 2024-09-18T00:41:14.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:14.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:14.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:14.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:14.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:41:14.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary8 2024-09-18T00:41:14.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:14.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary8 2024-09-18T00:41:14.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:14.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:41:14.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary9 2024-09-18T00:41:14.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:14.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:14.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:14.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:41:14.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+replaying 2024-09-18T00:41:14.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:14.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:14.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:14.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:14.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:41:14.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:41:14.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:14.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:41:15.354 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:41:15.351+0000 7f849c00d640 -1 reset not still connected to 0x5629a75ffe10 2024-09-18T00:41:15.360 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:41:15.355+0000 7f849c00d640 -1 reset not still connected to 0x5629a85e6f70 2024-09-18T00:41:15.360 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:41:15.355+0000 7f849c00d640 -1 reset not still connected to 0x5629a85e7040 2024-09-18T00:41:15.360 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:41:15.355+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:41:15.360 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:41:15.355+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:41:15.360 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:41:15.355+0000 7f849c00d640 -1 reset not still connected to 0x5629cbc1e270 2024-09-18T00:41:15.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+replaying '' '' 2024-09-18T00:41:15.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:15.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:15.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:15.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:15.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:41:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:41:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:41:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:41:15.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:41:15.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:41:15.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a3cef58b-69ac-4f11-993d-23070a4b6c51 2024-09-18T00:41:15.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:41:15.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62033920.0,"last_snapshot_bytes":62033920,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619930,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"} 2024-09-18T00:41:15.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:41:15.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:41:09 2024-09-18T00:41:15.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:41:15.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:41:15.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:41:15.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:41:15.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:41:14' 2024-09-18T00:41:15.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:41:15.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:41:15.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:41:15.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62033920.0,"last_snapshot_bytes":62033920,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619930,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"} 2024-09-18T00:41:15.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:41:15.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:41:15.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:41:15.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:41:15.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:41:15.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:41:15.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:41:15.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:41:15.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:41:09' 2024-09-18T00:41:15.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+replaying '2024-09-18 00:41:09' 2024-09-18T00:41:15.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:15.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:15.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:15.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:15.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:41:09' 2024-09-18T00:41:15.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:41:15.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:41:15.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:41:15.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10e4321661-07df-4164-962e-0d882a261febup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70250496.0,"last_snapshot_bytes":70250496,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:41:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:14image-primary9a3cef58b-69ac-4f11-993d-23070a4b6c51up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62033920.0,"last_snapshot_bytes":62033920,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619930,"remote_snapshot_timestamp":1726619930,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:41:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:14' 2024-09-18T00:41:15.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:41:15.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:41:15.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:41:09' 2024-09-18T00:41:15.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:41:15.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:41:15.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:41:15.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:41:15.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:41:15.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:15.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:15.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary9 2024-09-18T00:41:15.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:41:15.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:15.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:15.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary9 2024-09-18T00:41:15.948 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:15.943+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4000 2024-09-18T00:41:15.949 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:15.943+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4750 2024-09-18T00:41:15.955 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:15.951+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6734b1ee0 2024-09-18T00:41:15.955 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:15.951+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d68ccf2ea0 2024-09-18T00:41:15.955 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:15.951+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6a9188f70 2024-09-18T00:41:15.955 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:15.951+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4750 2024-09-18T00:41:15.958 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:15.955+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6734b1ee0 2024-09-18T00:41:15.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary9 2024-09-18T00:41:17.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:41:17.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary9 2024-09-18T00:41:17.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:17.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:17.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:17.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:17.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:41:17.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary9 2024-09-18T00:41:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:17.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary9 2024-09-18T00:41:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:41:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary10 2024-09-18T00:41:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:41:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+replaying 2024-09-18T00:41:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:17.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:17.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:17.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:17.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:41:17.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:41:17.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:17.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:41:18.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+replaying '' '' 2024-09-18T00:41:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:41:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:41:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:41:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:41:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:41:18.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:41:18.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: e4321661-07df-4164-962e-0d882a261feb 2024-09-18T00:41:18.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:41:18.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70250496.0,"last_snapshot_bytes":70250496,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"} 2024-09-18T00:41:18.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:41:18.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:41:09 2024-09-18T00:41:18.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:41:18.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:41:18.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:41:18.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:41:18.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:41:14' 2024-09-18T00:41:18.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:41:18.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:41:18.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:41:18.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70250496.0,"last_snapshot_bytes":70250496,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"} 2024-09-18T00:41:18.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:41:18.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:41:18.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:41:18.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:41:18.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:41:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:41:18.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:41:18.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:41:18.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:41:09' 2024-09-18T00:41:18.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+replaying '2024-09-18 00:41:09' 2024-09-18T00:41:18.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:18.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:18.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:18.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:41:18.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:41:09' 2024-09-18T00:41:18.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:41:18.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:41:18.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:41:18.979 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:18.979+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4000 2024-09-18T00:41:18.979 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:18.979+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4750 2024-09-18T00:41:18.984 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:18.983+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6734b1ee0 2024-09-18T00:41:18.984 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:18.983+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735afee0 2024-09-18T00:41:18.984 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:18.983+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d677191110 2024-09-18T00:41:18.984 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:41:18.983+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6a9188f70 2024-09-18T00:41:19.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10e4321661-07df-4164-962e-0d882a261febup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70250496.0,"last_snapshot_bytes":70250496,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726619936,"remote_snapshot_timestamp":1726619936,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:41:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:41:14' 2024-09-18T00:41:19.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:41:19.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:41:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:41:09' 2024-09-18T00:41:19.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:41:19.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:41:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:41:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:41:19.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:41:19.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:19.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:19.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary10 2024-09-18T00:41:19.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:41:19.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:19.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:19.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary10 2024-09-18T00:41:19.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary10 2024-09-18T00:41:20.329 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:41:20.327+0000 7fac32fac640 -1 reset not still connected to 0x55a839605ba0 2024-09-18T00:41:20.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:41:20.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary10 2024-09-18T00:41:20.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:20.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:20.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:20.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:41:20.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:41:20.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary10 2024-09-18T00:41:20.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:20.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary10 2024-09-18T00:41:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..10} 2024-09-18T00:41:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS=() 2024-09-18T00:41:20.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS=() 2024-09-18T00:41:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS=() 2024-09-18T00:41:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS=() 2024-09-18T00:41:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RET=0 2024-09-18T00:41:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:41:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary1 2024-09-18T00:41:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary1 snapshot 10G 2024-09-18T00:41:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:20.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary1 10G 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:41:20.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:41:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:20.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary1 2024-09-18T00:41:20.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:41:20.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:20.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:41:20.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:20.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary1 snapshot 2024-09-18T00:41:20.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:20.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:20.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:20.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:20.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary1 snapshot 2024-09-18T00:41:21.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:41:21.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary1 2024-09-18T00:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary1': 2024-09-18T00:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2bcdfe6d4d87 2024-09-18T00:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2bcdfe6d4d87 2024-09-18T00:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:41:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:41:20 2024 2024-09-18T00:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:41:20 2024 2024-09-18T00:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:41:20 2024 2024-09-18T00:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 1538ca69-026d-4504-986b-6ca3056a191b 2024-09-18T00:41:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:41:21.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:41:21.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:41:21.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd0 2024-09-18T00:41:21.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:41:21.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd0 2024-09-18T00:41:21.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:41:21.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:41:21.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:41:21.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 7b38e52e-3000-4cd3-bacf-3623aa2c168d 2024-09-18T00:41:21.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:41:21.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:41:21.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:21.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:41:21.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:41:21.685 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:41:21.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:21.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary1 2024-09-18T00:41:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd0 test-primary1 2024-09-18T00:41:21.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:41:21.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary1 2024-09-18T00:41:21.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:41:21.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary1 2024-09-18T00:41:21.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:21.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:21.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:21.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:21.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:41:21.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:21.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:21.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:21.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:41:21.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:41:21.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary1 2024-09-18T00:41:21.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:21.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary1 2024-09-18T00:41:21.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:41:21.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary2 2024-09-18T00:41:21.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary1 2024-09-18T00:41:21.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary2 snapshot 10G 2024-09-18T00:41:21.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:21.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:21.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:21.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary2 10G 2024-09-18T00:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:41:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:21.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:41:21.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:41:21.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:41:21.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:21.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary2 2024-09-18T00:41:21.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:41:21.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:21.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:41:21.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:21.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary2 snapshot 2024-09-18T00:41:21.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:21.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:21.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:41:21.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:21.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary2 snapshot 2024-09-18T00:41:22.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1986 2024-09-18T00:41:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:41:22.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:22.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary2 2024-09-18T00:41:22.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary2': 2024-09-18T00:41:22.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:41:22.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:41:22.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:41:22.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2bdccfb94e36 2024-09-18T00:41:22.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2bdccfb94e36 2024-09-18T00:41:22.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:41:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:41:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:41:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:41:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:41:21 2024 2024-09-18T00:41:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:41:21 2024 2024-09-18T00:41:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:41:21 2024 2024-09-18T00:41:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:41:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:41:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70 2024-09-18T00:41:22.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:41:22.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:41:22.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:41:22.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd1 2024-09-18T00:41:22.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:41:22.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd1 2024-09-18T00:41:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:41:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:41:22.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:41:22.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 43107885-2dc7-4cb8-bdaa-75d5b6aab93b 2024-09-18T00:41:22.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:41:22.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:41:22.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:22.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:41:22.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:41:22.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:41:22.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary1/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary1' 2024-09-18T00:41:22.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:41:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:41:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:22.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary2 2024-09-18T00:41:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd1 test-primary2 2024-09-18T00:41:22.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:41:22.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary2 2024-09-18T00:41:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:41:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary2 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary2 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary2 2024-09-18T00:41:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:41:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary3 2024-09-18T00:41:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary2 2024-09-18T00:41:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary3 snapshot 10G 2024-09-18T00:41:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary3 10G 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:41:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:41:23.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:41:23.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:23.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary3 2024-09-18T00:41:23.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:41:23.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:23.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:41:23.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:23.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary3 snapshot 2024-09-18T00:41:23.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:23.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:23.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:41:23.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:23.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary3 snapshot 2024-09-18T00:41:23.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:41:23.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary2/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary2' 2024-09-18T00:41:23.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1988 2024-09-18T00:41:23.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:41:23.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:23.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary3 2024-09-18T00:41:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary3': 2024-09-18T00:41:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:41:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:41:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:41:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2becaaeb90e 2024-09-18T00:41:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2becaaeb90e 2024-09-18T00:41:23.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:41:23.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:41:23.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:41:23.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:41:23.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:41:23 2024 2024-09-18T00:41:23.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:41:23 2024 2024-09-18T00:41:23.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:41:23 2024 2024-09-18T00:41:23.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:41:23.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:41:23.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 2cd3b2a7-cd9c-456d-b051-1ce989af2010 2024-09-18T00:41:23.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:41:23.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:41:23.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:41:24.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd2 2024-09-18T00:41:24.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:41:24.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd2 2024-09-18T00:41:24.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:41:24.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:41:24.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:41:24.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: e0184969-4668-420a-ab38-1c0231f3d5d9 2024-09-18T00:41:24.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:41:24.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:41:24.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:24.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:41:24.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:41:24.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:41:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:41:24.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:24.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary3 2024-09-18T00:41:24.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd2 test-primary3 2024-09-18T00:41:24.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:41:24.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary3 2024-09-18T00:41:24.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:41:24.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary3 2024-09-18T00:41:24.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:24.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:41:24.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:24.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:41:24.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:24.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary3 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary4 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary4 snapshot 10G 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary3 2024-09-18T00:41:24.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary3 2024-09-18T00:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary4 10G 2024-09-18T00:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:24.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:41:24.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:24.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:41:24.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:41:24.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:41:24.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:24.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary4 2024-09-18T00:41:24.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:41:24.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:24.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:41:24.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:24.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary4 snapshot 2024-09-18T00:41:24.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:24.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:41:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary4 snapshot 2024-09-18T00:41:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:41:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary3/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary3' 2024-09-18T00:41:25.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:25.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:41:25.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:25.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:25.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:25.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:41:25.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:41:25.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1990 2024-09-18T00:41:25.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary4 2024-09-18T00:41:25.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary4': 2024-09-18T00:41:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:41:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:41:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:41:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2bfb97e616e9 2024-09-18T00:41:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2bfb97e616e9 2024-09-18T00:41:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:41:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:41:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:41:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:41:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:41:24 2024 2024-09-18T00:41:25.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:41:24 2024 2024-09-18T00:41:25.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:41:24 2024 2024-09-18T00:41:25.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:41:25.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:41:25.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 4ddc12ed-652e-41cb-a869-7273142e4cee 2024-09-18T00:41:25.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:41:25.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:41:25.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:41:26.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd3 2024-09-18T00:41:26.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:41:26.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd3 2024-09-18T00:41:26.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:41:26.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:41:26.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:41:26.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 0505745e-6fa3-443e-82e9-c3fec42fcca8 2024-09-18T00:41:26.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:41:26.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:41:26.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:26.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:41:26.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:41:26.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:41:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:41:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:26.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary4 2024-09-18T00:41:26.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd3 test-primary4 2024-09-18T00:41:26.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:41:26.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary4 2024-09-18T00:41:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:41:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary4 2024-09-18T00:41:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:41:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:41:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary4 2024-09-18T00:41:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:41:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary5 2024-09-18T00:41:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary5 snapshot 10G 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary4 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary4 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:41:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:41:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary5 10G 2024-09-18T00:41:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:26.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:26.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:26.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:26.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:26.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:26.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:41:26.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:41:26.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:41:26.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:26.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary5 2024-09-18T00:41:26.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1992 2024-09-18T00:41:26.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:26.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:41:26.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:26.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:41:26.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:26.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary5 snapshot 2024-09-18T00:41:26.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:26.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:26.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:26.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:26.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary5 snapshot 2024-09-18T00:41:26.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:26.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:41:26.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:26.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:26.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:41:26.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:41:26.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:41:26.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary4/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary4' 2024-09-18T00:41:27.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1993 2024-09-18T00:41:27.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:41:27.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:27.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary5 2024-09-18T00:41:27.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary5': 2024-09-18T00:41:27.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2c0db894a464 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2c0db894a464 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:41:26 2024 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:41:26 2024 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:41:26 2024 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:41:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:41:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 4dabe928-cfdd-47d0-97f0-a3be00b4b2a0 2024-09-18T00:41:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:41:27.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:41:27.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:41:27.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd4 2024-09-18T00:41:27.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:41:27.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd4 2024-09-18T00:41:27.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:41:28.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:41:28.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:41:28.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 9ed4ff4f-02f3-4437-bb1f-e800a98495b6 2024-09-18T00:41:28.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:41:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:41:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:41:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:41:28.212 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:41:28.199+0000 7f849c00d640 -1 reset not still connected to 0x5629bda8f110 2024-09-18T00:41:28.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:41:28.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:41:28.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:28.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary5 2024-09-18T00:41:28.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd4 test-primary5 2024-09-18T00:41:28.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:41:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary5 2024-09-18T00:41:28.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:41:28.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary5 2024-09-18T00:41:28.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:28.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:41:28.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:28.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary6 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary5 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary6 snapshot 10G 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:28.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary5 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary6 10G 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary5 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:28.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:28.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:28.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:28.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:28.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:41:28.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:41:28.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:41:28.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:28.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary6 2024-09-18T00:41:28.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1995 2024-09-18T00:41:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:41:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:41:28.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:41:28.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:28.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:41:28.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:28.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:41:28.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:28.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary6 snapshot 2024-09-18T00:41:28.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:28.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:28.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:28.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:28.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary6 snapshot 2024-09-18T00:41:28.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:41:28.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary5/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary5' 2024-09-18T00:41:29.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:29.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:41:29.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:29.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:29.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:29.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:41:29.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1996 2024-09-18T00:41:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:29.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:41:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary6 2024-09-18T00:41:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary6': 2024-09-18T00:41:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:41:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:41:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:41:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2c2064309fb2 2024-09-18T00:41:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2c2064309fb2 2024-09-18T00:41:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:41:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:41:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:41:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:41:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:41:28 2024 2024-09-18T00:41:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:41:28 2024 2024-09-18T00:41:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:41:28 2024 2024-09-18T00:41:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:41:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:41:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 1595563f-88e0-4d3f-be52-370c261044d6 2024-09-18T00:41:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:41:30.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:41:30.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:41:30.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd5 2024-09-18T00:41:30.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:41:30.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd5 2024-09-18T00:41:30.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:41:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:41:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:41:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: f3d5a832-ad69-406f-a417-e7aed59d8cd7 2024-09-18T00:41:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:41:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:41:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:41:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:41:30.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): Snapshot ID: 1997 2024-09-18T00:41:30.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:30.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:41:30.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:30.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:30.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:41:30.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:41:30.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:30.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:41:31.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:41:31.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:31.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary6 2024-09-18T00:41:31.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd5 test-primary6 2024-09-18T00:41:31.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:41:31.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary6 2024-09-18T00:41:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:41:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary6 2024-09-18T00:41:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:41:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:41:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:41:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary6 2024-09-18T00:41:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:41:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary7 2024-09-18T00:41:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary7 snapshot 10G 2024-09-18T00:41:31.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:31.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary6 2024-09-18T00:41:31.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:31.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:31.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary6 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary7 10G 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:41:31.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:41:31.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:41:31.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:31.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary7 2024-09-18T00:41:31.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:41:31.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:31.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:41:31.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:31.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary7 snapshot 2024-09-18T00:41:31.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:31.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:31.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:31.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:31.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary7 snapshot 2024-09-18T00:41:31.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:41:31.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary6/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary6' 2024-09-18T00:41:31.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:31.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:41:31.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:31.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:31.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:41:31.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:41:31.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 1999 2024-09-18T00:41:31.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2001 2024-09-18T00:41:31.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:41:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:31.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary7 2024-09-18T00:41:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary7': 2024-09-18T00:41:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:41:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:41:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:41:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2c397ad2c70e 2024-09-18T00:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2c397ad2c70e 2024-09-18T00:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:41:31 2024 2024-09-18T00:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:41:31 2024 2024-09-18T00:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:41:31 2024 2024-09-18T00:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: ab0d841f-81f9-4384-b3bd-341cb0d2c745 2024-09-18T00:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:41:32.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:41:32.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:41:32.223 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:41:32.211+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:41:32.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd6 2024-09-18T00:41:32.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:41:32.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd6 2024-09-18T00:41:32.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:41:32.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:41:32.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:41:32.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: e43fb11f-ae6f-4807-a885-a727f786c0d9 2024-09-18T00:41:32.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:41:32.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:41:32.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:32.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:41:32.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:41:32.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:41:32.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 Snapshot ID: 2000 2024-09-18T00:41:32.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:32.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:41:32.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:32.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary7 2024-09-18T00:41:32.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd6 test-primary7 2024-09-18T00:41:32.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:32.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:41:32.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:32.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:32.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:32.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:41:32.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:41:32.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary7 2024-09-18T00:41:32.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:41:32.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary7 2024-09-18T00:41:32.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:32.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:41:32.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:32.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:32.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:32.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:32.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:41:32.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary7 2024-09-18T00:41:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary8 2024-09-18T00:41:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:41:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary8 snapshot 10G 2024-09-18T00:41:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary8 10G 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary7 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary7 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:41:32.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:41:32.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:41:32.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:32.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary8 2024-09-18T00:41:33.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:41:33.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:33.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:41:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary8 snapshot 2024-09-18T00:41:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary8 snapshot 2024-09-18T00:41:33.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:41:33.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary7/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary7' 2024-09-18T00:41:33.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2003 2024-09-18T00:41:33.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:33.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:41:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2005 2024-09-18T00:41:33.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:41:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:41:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:41:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary8 2024-09-18T00:41:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary8': 2024-09-18T00:41:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:41:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:41:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:41:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2c4b64a6cfa0 2024-09-18T00:41:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2c4b64a6cfa0 2024-09-18T00:41:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:41:33.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:41:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:41:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:41:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:41:33 2024 2024-09-18T00:41:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:41:33 2024 2024-09-18T00:41:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:41:33 2024 2024-09-18T00:41:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:41:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:41:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 8f3bd77d-71f5-46c9-8ccc-36529855a3b4 2024-09-18T00:41:33.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:41:33.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:41:33.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:41:34.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd7 2024-09-18T00:41:34.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:41:34.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd7 2024-09-18T00:41:34.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:41:34.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:41:34.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:41:34.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: a1247303-2eac-4300-8663-cd7b35aaa1a7 2024-09-18T00:41:34.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:41:34.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:41:34.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:34.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:41:34.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:41:34.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:41:34.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:41:34.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:34.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary8 2024-09-18T00:41:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd7 test-primary8 2024-09-18T00:41:34.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:41:34.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary8 2024-09-18T00:41:34.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:41:34.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary8 2024-09-18T00:41:34.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:34.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary8 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary9 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary9 snapshot 10G 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary8 2024-09-18T00:41:34.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary8 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary9 10G 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:34.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:34.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:34.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:41:34.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:41:34.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:41:34.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:34.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary9 2024-09-18T00:41:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:41:34.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:41:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary9 snapshot 2024-09-18T00:41:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:34.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary9 snapshot 2024-09-18T00:41:34.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:41:34.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary8/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary8' 2024-09-18T00:41:34.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2004 2024-09-18T00:41:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:34.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:34.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:41:34.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:34.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:34.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:34.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:41:34.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:41:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:34.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:41:35.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:35.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:41:35.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:35.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:35.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:41:35.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:41:35.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2007 2024-09-18T00:41:35.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:35.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:41:35.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary9 2024-09-18T00:41:35.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2008 2024-09-18T00:41:35.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary9': 2024-09-18T00:41:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:41:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:41:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:41:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2c5e57876b12 2024-09-18T00:41:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2c5e57876b12 2024-09-18T00:41:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:41:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:41:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:41:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:41:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:41:34 2024 2024-09-18T00:41:35.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:41:34 2024 2024-09-18T00:41:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:41:34 2024 2024-09-18T00:41:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:41:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:41:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 47cdb799-ea3d-491b-a019-2a4da9cb8764 2024-09-18T00:41:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:41:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:41:35.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:41:36.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd8 2024-09-18T00:41:36.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:41:36.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd8 2024-09-18T00:41:36.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:41:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:41:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:41:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 22d85633-ea3e-4a36-8161-79cf9dfe0447 2024-09-18T00:41:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:41:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:41:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:41:36.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:41:36.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:41:36.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:41:36.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary9 2024-09-18T00:41:36.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd8 test-primary9 2024-09-18T00:41:36.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:36.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:41:36.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:36.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:36.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:41:36.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:41:36.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:36.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:41:36.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:36.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:36.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:36.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:41:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:41:36.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary9 2024-09-18T00:41:36.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2011 2024-09-18T00:41:36.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2010 2024-09-18T00:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary9 2024-09-18T00:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:41:36.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:41:36.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary10 2024-09-18T00:41:36.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary10 snapshot 10G 2024-09-18T00:41:36.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:36.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:36.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:36.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:36.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary9 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary10 10G 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary9 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary9 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary10 2024-09-18T00:41:36.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:36.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:37.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:41:37.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary9/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary9' 2024-09-18T00:41:37.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:41:37.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:37.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:41:37.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:41:37.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary10 snapshot 2024-09-18T00:41:37.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:37.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:37.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:37.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:41:37.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary10 snapshot 2024-09-18T00:41:37.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2015 2024-09-18T00:41:37.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:37.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2012 2024-09-18T00:41:37.798 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:41:37.799+0000 7f849c00d640 -1 reset not still connected to 0x5629de3c3a00 2024-09-18T00:41:37.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:37.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:41:37.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:37.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:37.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:37.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:41:37.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:37.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:41:37.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary10 2024-09-18T00:41:37.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary10': 2024-09-18T00:41:37.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:41:37.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:41:37.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:41:37.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 2c7a93e9904a 2024-09-18T00:41:37.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.2c7a93e9904a 2024-09-18T00:41:37.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:41:37.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:41:37.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:41:37.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:41:37.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:41:37 2024 2024-09-18T00:41:37.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:41:37 2024 2024-09-18T00:41:37.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:41:37 2024 2024-09-18T00:41:37.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:41:37.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:41:37.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 9b75893a-9ef9-468b-ad0c-88ee663b581a 2024-09-18T00:41:37.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:41:37.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:41:37.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:41:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd9 2024-09-18T00:41:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:41:38.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd9 2024-09-18T00:41:38.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:41:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:41:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:41:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: e08ab2b8-06e7-471c-834b-12a4611b4cdc 2024-09-18T00:41:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:41:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:41:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:41:38.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:41:38.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:41:38.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:41:38.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:41:38.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary10 2024-09-18T00:41:38.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd9 test-primary10 2024-09-18T00:41:38.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2013 2024-09-18T00:41:38.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:38.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:41:38.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:38.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:38.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:41:38.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:41:38.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:38.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2014 2024-09-18T00:41:38.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:41:38.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary10 2024-09-18T00:41:38.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:41:38.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:41:38.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:41:38.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 97442 2024-09-18T00:41:38.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary10 2024-09-18T00:41:38.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:38.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary10 2024-09-18T00:41:38.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:38.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:38.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:38.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:41:38.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:38.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:38.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:38.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:41:38.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary10 2024-09-18T00:41:38.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary10 2024-09-18T00:41:38.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:41:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:41:39.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:41:39.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary10/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary10' 2024-09-18T00:41:39.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2017 2024-09-18T00:41:39.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:39.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2019 2024-09-18T00:41:39.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:39.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:39.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:41:39.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:39.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:39.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:39.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:41:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:41:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:41:40.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:40.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:41:40.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:40.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:40.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:40.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:41:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:41:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:41:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:41:40.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2018 2024-09-18T00:41:40.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2020 2024-09-18T00:41:40.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:40.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:41.581 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:41:41.578+0000 7fd4c158b640 -1 reset not still connected to 0x56436a82e340 2024-09-18T00:41:41.581 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:41:41.578+0000 7fd4c158b640 -1 reset not still connected to 0x56436b6aeb60 2024-09-18T00:41:41.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:41.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:41:41.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:41.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:41.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:41:41.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:41:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2021 2024-09-18T00:41:41.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2022 2024-09-18T00:41:41.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:41.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:41.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:41:41.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:41.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:41.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:41.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:41:41.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:42.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2023 2024-09-18T00:41:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:42.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2024 2024-09-18T00:41:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:42.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:42.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:41:42.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:42.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:42.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:42.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:41:42.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:42.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:41:42.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:42.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:41:43.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2026 2024-09-18T00:41:43.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2025 2024-09-18T00:41:43.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:43.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:41:43.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:43.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:43.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:43.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:41:43.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:43.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:43.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:41:43.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:43.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:43.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:41:43.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:41:43.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:44.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2027 2024-09-18T00:41:44.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:44.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:41:44.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:44.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:44.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:44.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:41:44.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2028 2024-09-18T00:41:44.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:44.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:44.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:41:44.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:44.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:44.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:44.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:41:45.007 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:41:45.002+0000 7f849c00d640 -1 reset not still connected to 0x5629bbcc9ba0 2024-09-18T00:41:45.008 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:41:45.006+0000 7f849c00d640 -1 reset not still connected to 0x5629d36244e0 2024-09-18T00:41:45.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:45.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:41:45.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:45.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:45.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:45.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:41:45.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:45.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:41:45.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:45.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:45.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:41:45.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:41:45.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2029 2024-09-18T00:41:45.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:45.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2030 2024-09-18T00:41:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:46.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2031 2024-09-18T00:41:46.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2032 2024-09-18T00:41:46.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:46.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:41:46.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:46.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:46.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:46.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:41:46.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:46.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:46.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:41:46.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:46.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:41:46.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:47.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2034 2024-09-18T00:41:47.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2033 2024-09-18T00:41:47.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:47.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:47.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:47.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:41:47.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:47.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:47.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:47.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:41:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:41:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:41:48.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2036 2024-09-18T00:41:48.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:48.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2035 2024-09-18T00:41:48.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:48.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:41:48.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:48.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:48.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:48.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:41:48.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:41:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:41:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:41:49.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2037 2024-09-18T00:41:49.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:49.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:41:49.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:49.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:49.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:49.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:41:49.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:49.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2038 2024-09-18T00:41:49.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:49.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:41:49.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:49.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:49.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:49.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:41:49.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:50.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:50.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:41:50.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:50.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:50.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:41:50.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:41:50.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:41:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:50.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:41:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2040 2024-09-18T00:41:50.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:50.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2039 2024-09-18T00:41:50.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:51.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2042 2024-09-18T00:41:51.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:51.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:41:51.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:51.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:51.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:41:51.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:41:51.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2041 2024-09-18T00:41:51.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:51.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:51.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:51.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:41:51.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:51.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:51.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:51.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:41:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2043 2024-09-18T00:41:52.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:52.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2044 2024-09-18T00:41:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:52.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:52.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:41:52.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:52.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:52.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:52.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:41:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:41:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:52.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:41:53.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2046 2024-09-18T00:41:53.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:53.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2045 2024-09-18T00:41:53.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:53.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:53.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:41:53.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:53.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:53.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:41:53.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:41:53.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:53.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:41:53.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:53.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:53.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:53.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:41:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:41:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2047 2024-09-18T00:41:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:41:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:54.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:54.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:54.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:41:54.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:54.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2048 2024-09-18T00:41:55.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:55.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:55.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:41:55.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:55.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:55.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:41:55.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:41:55.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:55.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:41:55.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:55.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:55.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:41:55.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:41:55.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2049 2024-09-18T00:41:55.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2050 2024-09-18T00:41:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:55.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:56.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:56.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:41:56.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:56.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:56.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:41:56.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:41:56.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:56.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:41:56.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:56.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:56.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:41:56.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:41:56.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2051 2024-09-18T00:41:56.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2052 2024-09-18T00:41:56.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:56.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:57.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2053 2024-09-18T00:41:57.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2054 2024-09-18T00:41:57.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:57.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:57.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:57.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:41:57.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:57.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:57.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:41:57.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:41:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:58.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:41:58.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:58.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:58.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:41:58.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:41:58.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2056 2024-09-18T00:41:58.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2055 2024-09-18T00:41:58.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:58.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:41:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:41:58.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:41:58.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:58.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:41:58.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:58.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:58.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:41:58.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:41:59.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2058 2024-09-18T00:41:59.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2057 2024-09-18T00:41:59.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:59.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:41:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:41:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:41:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:41:59.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:41:59.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:41:59.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:41:59.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:41:59.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:41:59.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:00.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2059 2024-09-18T00:42:00.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2060 2024-09-18T00:42:00.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:00.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:42:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:00.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:42:00.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:42:00.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:00.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:42:00.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:00.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:00.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:42:00.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:42:01.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2061 2024-09-18T00:42:01.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2062 2024-09-18T00:42:01.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:01.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:42:01.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:01.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:01.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:42:01.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:42:01.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:01.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:01.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:01.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:42:01.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:01.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:01.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:42:01.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:42:02.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2063 2024-09-18T00:42:02.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:42:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:42:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:42:02.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2064 2024-09-18T00:42:02.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:02.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:42:02.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:02.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:02.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:42:02.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:42:02.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:02.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:03.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:03.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:42:03.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:03.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:03.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:42:03.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:42:03.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:03.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:42:03.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:03.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:03.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:42:03.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:42:03.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2065 2024-09-18T00:42:03.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2066 2024-09-18T00:42:03.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:03.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:04.439 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:04.433+0000 7f52d7c6c640 -1 reset not still connected to 0x557871fd5380 2024-09-18T00:42:04.439 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:04.433+0000 7f52d7c6c640 -1 reset not still connected to 0x557872bb1380 2024-09-18T00:42:04.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:42:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:42:04.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:42:04.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:04.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:42:04.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:04.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:04.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:42:04.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:04.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2067 2024-09-18T00:42:04.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2068 2024-09-18T00:42:04.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:04.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:42:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:05.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:42:05.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:42:05.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:05.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:42:05.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:05.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:05.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:42:05.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:42:05.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2069 2024-09-18T00:42:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2070 2024-09-18T00:42:05.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:05.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:06.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2072 2024-09-18T00:42:06.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2071 2024-09-18T00:42:06.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:06.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:42:06.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:06.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:06.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:42:06.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:42:06.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:06.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:42:06.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:06.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:06.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:42:06.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:42:06.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:07.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2073 2024-09-18T00:42:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2074 2024-09-18T00:42:07.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:07.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:42:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:42:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:42:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:07.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:42:07.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:07.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:07.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:42:07.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:42:08.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2076 2024-09-18T00:42:08.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2075 2024-09-18T00:42:08.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:08.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:42:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:42:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:42:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:42:08.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:42:09.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2077 2024-09-18T00:42:09.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2078 2024-09-18T00:42:09.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:09.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:42:09.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:09.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:09.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:42:09.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:09.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:09.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:42:09.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:09.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:09.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:42:09.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:42:09.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:09.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:10.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:10.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:42:10.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:10.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:10.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:42:10.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:42:10.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:10.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:42:10.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:10.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:10.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:42:10.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:42:10.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2080 2024-09-18T00:42:10.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2079 2024-09-18T00:42:10.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:11.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2082 2024-09-18T00:42:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:11.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:11.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:42:11.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:11.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:11.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:42:11.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:42:11.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:11.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:42:11.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:11.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:11.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:42:11.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:42:11.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2081 2024-09-18T00:42:11.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2084 2024-09-18T00:42:12.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2083 2024-09-18T00:42:12.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:12.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:12.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:42:12.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:12.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:12.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:12.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:42:12.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:42:12.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:42:12.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:12.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:12.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:42:12.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:42:13.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2085 2024-09-18T00:42:13.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:13.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:42:13.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:13.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:13.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:42:13.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:42:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:42:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:42:13.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:42:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:13.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2086 2024-09-18T00:42:13.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:14.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:14.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:42:14.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:14.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:14.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:42:14.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:14.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2088 2024-09-18T00:42:14.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2087 2024-09-18T00:42:14.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:14.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:14.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:14.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:42:14.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:14.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:14.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:42:14.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:42:15.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:15.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:42:15.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:15.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:15.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:42:15.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:42:15.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:15.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:42:15.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:15.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:15.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:42:15.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:42:15.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2089 2024-09-18T00:42:15.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2090 2024-09-18T00:42:15.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:15.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:15.921 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:42:15.917+0000 7f849c00d640 -1 reset not still connected to 0x5629e3328c30 2024-09-18T00:42:16.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2092 2024-09-18T00:42:16.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2091 2024-09-18T00:42:16.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:42:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:42:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:42:16.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:16.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:16.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:16.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:42:16.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:16.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:16.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:42:16.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:42:17.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2093 2024-09-18T00:42:17.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2094 2024-09-18T00:42:17.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:17.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:17.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:17.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:42:17.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:17.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:17.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:42:17.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:42:17.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:42:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:42:17.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:42:18.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:18.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:42:18.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:18.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:18.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:42:18.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:42:18.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:18.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:42:18.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:18.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:18.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:42:18.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:42:18.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2095 2024-09-18T00:42:18.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2096 2024-09-18T00:42:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:18.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:19.593 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:19.589+0000 7fd4c158b640 -1 reset not still connected to 0x5643587075f0 2024-09-18T00:42:19.597 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:19.593+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a340 2024-09-18T00:42:19.597 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:19.593+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T00:42:19.597 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:19.593+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T00:42:19.598 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:19.593+0000 7fd4c158b640 -1 reset not still connected to 0x56436a82e340 2024-09-18T00:42:19.598 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:19.593+0000 7fd4c158b640 -1 reset not still connected to 0x56436b08c1a0 2024-09-18T00:42:19.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:19.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:42:19.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:19.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:19.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:42:19.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:42:19.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2098 2024-09-18T00:42:19.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2097 2024-09-18T00:42:19.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:19.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:42:19.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:19.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:19.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:42:19.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:19.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:19.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:20.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:20.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:42:20.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:20.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:20.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:42:20.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:42:20.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:20.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:42:20.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:20.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:20.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:42:20.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:42:20.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2099 2024-09-18T00:42:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2100 2024-09-18T00:42:20.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:20.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:21.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2102 2024-09-18T00:42:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:42:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:42:21.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:42:21.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:21.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:42:21.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:21.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:21.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:42:21.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:42:21.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2101 2024-09-18T00:42:21.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:22.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2104 2024-09-18T00:42:22.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:22.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2103 2024-09-18T00:42:22.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:22.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:22.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:42:22.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:22.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:22.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:42:22.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:42:22.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:22.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:42:22.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:22.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:22.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:42:22.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:42:23.662 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:23.657+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T00:42:23.662 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:23.657+0000 7f52d7c6c640 -1 reset not still connected to 0x557843cf0c30 2024-09-18T00:42:23.662 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:23.657+0000 7f52d7c6c640 -1 reset not still connected to 0x557848243110 2024-09-18T00:42:23.662 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:23.657+0000 7f52d7c6c640 -1 reset not still connected to 0x55785ba60270 2024-09-18T00:42:23.662 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:23.657+0000 7f52d7c6c640 -1 reset not still connected to 0x557871fd5380 2024-09-18T00:42:23.662 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:23.657+0000 7f52d7c6c640 -1 reset not still connected to 0x557872bb1380 2024-09-18T00:42:23.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2106 2024-09-18T00:42:23.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:23.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:42:23.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:23.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:23.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:42:23.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:42:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2105 2024-09-18T00:42:23.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:23.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:42:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:42:23.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:42:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:24.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2108 2024-09-18T00:42:24.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2107 2024-09-18T00:42:24.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:24.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:42:24.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:24.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:24.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:42:24.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:24.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:24.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:24.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:24.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:42:24.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:24.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:24.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:42:24.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:42:25.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:25.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:42:25.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:25.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:25.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:42:25.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:42:25.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:25.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:42:25.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:25.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:25.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:42:25.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:42:25.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2110 2024-09-18T00:42:25.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2109 2024-09-18T00:42:25.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:25.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:26.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2111 2024-09-18T00:42:26.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2112 2024-09-18T00:42:26.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:26.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:42:26.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:26.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:26.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:42:26.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:42:26.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:26.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:42:26.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:26.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:26.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:42:26.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:42:26.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:26.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:27.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2114 2024-09-18T00:42:27.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2113 2024-09-18T00:42:27.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:27.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:42:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:42:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:42:27.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:27.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:42:27.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:27.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:27.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:42:27.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:42:28.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2116 2024-09-18T00:42:28.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2115 2024-09-18T00:42:28.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:28.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:42:28.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:28.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:28.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:42:28.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:42:28.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:28.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:42:28.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:28.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:28.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:42:28.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:42:28.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:28.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:29.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2117 2024-09-18T00:42:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2118 2024-09-18T00:42:29.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:29.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:29.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:42:29.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:29.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:29.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:42:29.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:29.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:29.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:29.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:42:29.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:29.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:29.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:42:29.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:42:30.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:30.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:42:30.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:30.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:30.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:42:30.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:42:30.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:30.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:42:30.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:30.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:30.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:42:30.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:42:30.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2119 2024-09-18T00:42:30.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:30.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2120 2024-09-18T00:42:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:31.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2122 2024-09-18T00:42:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:42:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:31.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:42:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:42:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:42:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:42:31.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:42:31.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2121 2024-09-18T00:42:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:31.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:32.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2123 2024-09-18T00:42:32.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2124 2024-09-18T00:42:32.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:32.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:32.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:32.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:42:32.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:42:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:42:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:42:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:42:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:42:33.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2126 2024-09-18T00:42:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2125 2024-09-18T00:42:33.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:33.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:33.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:42:33.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:33.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:33.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:42:33.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:42:33.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:33.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:42:33.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:33.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:33.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:42:33.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:42:34.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2128 2024-09-18T00:42:34.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2127 2024-09-18T00:42:34.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:34.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:42:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:42:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:42:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:34.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:42:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:42:35.467 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:35.461+0000 7f52d7c6c640 -1 reset not still connected to 0x557848243110 2024-09-18T00:42:35.467 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:35.461+0000 7f52d7c6c640 -1 reset not still connected to 0x55785ba60270 2024-09-18T00:42:35.467 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:35.461+0000 7f52d7c6c640 -1 reset not still connected to 0x557871fd5380 2024-09-18T00:42:35.467 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:35.461+0000 7f52d7c6c640 -1 reset not still connected to 0x557872bb1380 2024-09-18T00:42:35.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:35.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:42:35.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:35.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:35.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:42:35.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:42:35.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2129 2024-09-18T00:42:35.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:35.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:42:35.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:35.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:35.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:42:35.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:42:35.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2130 2024-09-18T00:42:35.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:35.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:36.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2131 2024-09-18T00:42:36.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2132 2024-09-18T00:42:36.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:36.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:42:36.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:36.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:36.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:42:36.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:42:36.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:36.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:36.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:36.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:42:36.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:36.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:36.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:42:36.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:42:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2133 2024-09-18T00:42:37.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2134 2024-09-18T00:42:37.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:37.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:37.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:37.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:42:37.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:37.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:37.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:42:37.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:42:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:42:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:37.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:37.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:42:37.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:42:38.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2136 2024-09-18T00:42:38.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:42:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:42:38.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:42:38.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2135 2024-09-18T00:42:38.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:38.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:42:38.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:38.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:38.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:42:38.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:42:38.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:39.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:39.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:42:39.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:39.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:39.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:42:39.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:42:39.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:39.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:42:39.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:39.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:39.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:42:39.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:39.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2137 2024-09-18T00:42:39.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2138 2024-09-18T00:42:39.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:39.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:40.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:40.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:42:40.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:40.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:40.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:42:40.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:42:40.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:40.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:42:40.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:40.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:40.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:42:40.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:42:40.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2139 2024-09-18T00:42:40.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2140 2024-09-18T00:42:40.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:40.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:41.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2142 2024-09-18T00:42:41.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:41.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:42:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:42:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:42:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:42:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:42:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:42:41.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2141 2024-09-18T00:42:42.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:42.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2144 2024-09-18T00:42:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2143 2024-09-18T00:42:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:42:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:42:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:42:42.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:42.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:42:42.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:42.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:42.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:42:42.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:42:43.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2146 2024-09-18T00:42:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2145 2024-09-18T00:42:43.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:42:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:42:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:42:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:42:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:42:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:42:43.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:44.500 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:44.492+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T00:42:44.500 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:44.492+0000 7f52d7c6c640 -1 reset not still connected to 0x557843cf0c30 2024-09-18T00:42:44.500 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:44.496+0000 7f52d7c6c640 -1 reset not still connected to 0x55785ba60270 2024-09-18T00:42:44.501 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:44.496+0000 7f52d7c6c640 -1 reset not still connected to 0x557871fd5380 2024-09-18T00:42:44.501 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:42:44.496+0000 7f52d7c6c640 -1 reset not still connected to 0x557872bb1380 2024-09-18T00:42:44.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:42:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:42:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:42:44.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2147 2024-09-18T00:42:44.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2148 2024-09-18T00:42:44.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:44.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:45.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:45.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:42:45.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:45.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:45.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:42:45.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:45.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:45.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:42:45.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:45.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:45.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:45.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:42:45.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:42:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:42:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:42:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:42:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2150 2024-09-18T00:42:45.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:45.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2149 2024-09-18T00:42:45.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:46.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2151 2024-09-18T00:42:46.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:46.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:42:46.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:46.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:46.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:42:46.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:42:46.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2152 2024-09-18T00:42:46.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:46.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:46.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:42:46.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:46.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:46.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:42:46.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:42:46.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:47.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2154 2024-09-18T00:42:47.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2153 2024-09-18T00:42:47.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:47.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:42:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:42:47.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:42:47.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:47.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:42:47.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:47.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:47.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:42:47.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:42:48.488 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:48.480+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a340 2024-09-18T00:42:48.488 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:48.480+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T00:42:48.489 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:48.480+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T00:42:48.489 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:48.480+0000 7fd4c158b640 -1 reset not still connected to 0x56436a82e340 2024-09-18T00:42:48.494 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:48.492+0000 7fd4c158b640 -1 reset not still connected to 0x56436b6aeb60 2024-09-18T00:42:48.516 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:48.508+0000 7fd4c158b640 -1 reset not still connected to 0x56436a82e340 2024-09-18T00:42:48.518 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:48.512+0000 7fd4c158b640 -1 reset not still connected to 0x56436b08c1a0 2024-09-18T00:42:48.518 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:42:48.512+0000 7fd4c158b640 -1 reset not still connected to 0x56436b6aeb60 2024-09-18T00:42:48.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:48.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:42:48.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:48.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:48.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:42:48.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:42:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2156 2024-09-18T00:42:48.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2155 2024-09-18T00:42:48.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:48.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:48.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:48.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:42:48.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:48.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:48.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:42:48.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:42:49.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2157 2024-09-18T00:42:49.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2158 2024-09-18T00:42:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:42:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:42:49.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:42:49.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:49.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:42:49.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:49.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:49.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:42:49.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:49.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:49.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:42:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:42:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:42:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:42:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:42:50.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:42:50.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2159 2024-09-18T00:42:50.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2160 2024-09-18T00:42:50.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:50.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:50.889 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:42:50.884+0000 7f849c00d640 -1 reset not still connected to 0x5629e3328c30 2024-09-18T00:42:51.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:51.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:42:51.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:51.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:51.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:42:51.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:42:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2162 2024-09-18T00:42:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:42:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:42:51.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:42:51.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2161 2024-09-18T00:42:51.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:51.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:52.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2164 2024-09-18T00:42:52.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2163 2024-09-18T00:42:52.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:52.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:42:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:42:52.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:42:52.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:52.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:42:52.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:52.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:52.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:42:52.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:42:53.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2165 2024-09-18T00:42:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:42:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:42:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:42:53.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2166 2024-09-18T00:42:53.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:53.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:42:53.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:53.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:53.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:42:53.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:42:53.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:53.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:54.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2167 2024-09-18T00:42:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2168 2024-09-18T00:42:54.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:54.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:42:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:42:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:42:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:42:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:54.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:42:54.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:55.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:55.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:42:55.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:55.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:55.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:42:55.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:42:55.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:55.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:42:55.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:55.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:55.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:42:55.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:42:55.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2169 2024-09-18T00:42:55.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2170 2024-09-18T00:42:55.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:55.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:56.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2172 2024-09-18T00:42:56.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:56.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:42:56.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:56.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:56.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:42:56.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:42:56.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2171 2024-09-18T00:42:56.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:56.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:42:56.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:56.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:56.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:42:56.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:42:56.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:56.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:57.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2174 2024-09-18T00:42:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2173 2024-09-18T00:42:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:57.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:57.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:57.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:42:57.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:57.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:57.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:42:57.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:42:57.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:57.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:42:57.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:57.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:57.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:42:57.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:42:58.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2175 2024-09-18T00:42:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2176 2024-09-18T00:42:58.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:42:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:42:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:42:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:58.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:58.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:58.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:42:58.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:58.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:58.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:42:58.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:42:59.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2178 2024-09-18T00:42:59.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2177 2024-09-18T00:42:59.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:59.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:42:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:42:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:42:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:42:59.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:42:59.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:42:59.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:42:59.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:42:59.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:42:59.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:43:00.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:00.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:43:00.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:00.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:00.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:43:00.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:43:00.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:00.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:43:00.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:00.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:00.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:43:00.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:43:00.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2180 2024-09-18T00:43:00.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2179 2024-09-18T00:43:00.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:00.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:01.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2181 2024-09-18T00:43:01.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2182 2024-09-18T00:43:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:43:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:43:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:43:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:43:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:43:01.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:43:01.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:01.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:02.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2183 2024-09-18T00:43:02.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2184 2024-09-18T00:43:02.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:02.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:02.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:02.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:02.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:43:02.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:43:02.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:02.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:02.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:02.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:02.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:43:02.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:43:02.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:43:02.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:43:03.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2185 2024-09-18T00:43:03.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2186 2024-09-18T00:43:03.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:03.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:43:03.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:03.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:03.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:43:03.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:43:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:43:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:43:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:43:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:03.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:04.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2188 2024-09-18T00:43:04.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:04.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:43:04.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:04.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:04.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:43:04.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:43:04.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:04.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2187 2024-09-18T00:43:04.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:04.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:43:04.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:04.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:04.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:43:04.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:43:04.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:05.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:05.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:43:05.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:05.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:05.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:43:05.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:43:05.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:05.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:43:05.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:05.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:05.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:43:05.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:43:05.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2189 2024-09-18T00:43:05.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2190 2024-09-18T00:43:05.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:05.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:06.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2192 2024-09-18T00:43:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:43:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:43:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:43:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:43:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:43:06.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:43:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2191 2024-09-18T00:43:06.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2193 2024-09-18T00:43:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2194 2024-09-18T00:43:07.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:07.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:43:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:43:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:43:07.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:07.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:43:07.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:07.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:07.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:43:07.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:43:08.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2195 2024-09-18T00:43:08.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2196 2024-09-18T00:43:08.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:08.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:08.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:08.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:43:08.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:08.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:08.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:43:08.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:43:08.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:08.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:43:08.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:08.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:08.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:43:08.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:43:09.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2197 2024-09-18T00:43:09.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:09.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2198 2024-09-18T00:43:09.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:09.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:09.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:43:09.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:09.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:09.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:43:09.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:43:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:43:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:43:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:43:10.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2199 2024-09-18T00:43:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:43:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:10.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:10.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:43:10.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:43:10.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:10.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:10.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:43:10.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:10.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:10.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:43:10.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:43:10.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2200 2024-09-18T00:43:10.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:11.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2201 2024-09-18T00:43:11.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:11.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:11.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:43:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:43:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:43:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:43:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:43:11.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:43:11.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2202 2024-09-18T00:43:11.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:43:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:43:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:43:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2204 2024-09-18T00:43:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2203 2024-09-18T00:43:12.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:12.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:43:12.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:12.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:12.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:43:12.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:43:12.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:12.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2205 2024-09-18T00:43:13.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:43:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:13.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:43:13.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:43:13.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2206 2024-09-18T00:43:13.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:13.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:43:13.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:13.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:13.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:43:13.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:43:13.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:43:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:43:14.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:43:14.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2207 2024-09-18T00:43:14.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:14.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:43:14.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:14.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:14.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:43:14.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:43:14.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2208 2024-09-18T00:43:14.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:14.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:15.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2209 2024-09-18T00:43:15.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:15.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:43:15.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:15.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:15.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:43:15.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:43:15.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:15.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:43:15.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:15.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:15.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:43:15.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:43:15.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2210 2024-09-18T00:43:15.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:15.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:16.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:16.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:43:16.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:16.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:16.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:43:16.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:43:16.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2211 2024-09-18T00:43:16.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:16.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:43:16.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:16.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:16.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:43:16.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:43:16.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:16.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2212 2024-09-18T00:43:16.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2213 2024-09-18T00:43:17.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2214 2024-09-18T00:43:17.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:17.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:17.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:17.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:43:17.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:17.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:17.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:43:17.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:43:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:43:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:43:17.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:43:18.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2216 2024-09-18T00:43:18.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2215 2024-09-18T00:43:18.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:18.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:18.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:43:18.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:18.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:18.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:43:18.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:43:18.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:18.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:18.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:43:18.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:18.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:18.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:43:18.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:43:19.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2218 2024-09-18T00:43:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2217 2024-09-18T00:43:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:43:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:43:19.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:43:19.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:19.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:19.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:19.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:43:19.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:19.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:19.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:43:19.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:43:20.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:20.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:43:20.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:20.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:20.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:43:20.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:43:20.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:20.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:43:20.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:20.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:20.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:43:20.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:43:20.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2219 2024-09-18T00:43:20.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2220 2024-09-18T00:43:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:20.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:21.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2221 2024-09-18T00:43:21.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:21.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:43:21.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:21.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:21.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:43:21.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:43:21.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:21.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2222 2024-09-18T00:43:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:43:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:43:21.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:43:21.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:22.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2223 2024-09-18T00:43:22.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:22.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2224 2024-09-18T00:43:22.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:22.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:22.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:43:22.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:22.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:22.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:43:22.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:43:22.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:22.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:43:22.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:22.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:22.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:43:22.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:43:23.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2225 2024-09-18T00:43:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:43:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:23.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:23.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:43:23.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:43:23.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:23.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:23.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:43:23.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:23.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:23.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:43:23.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:43:23.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2226 2024-09-18T00:43:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:24.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:24.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:43:24.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:24.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:24.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:43:24.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:43:24.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2227 2024-09-18T00:43:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:43:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:43:24.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:43:24.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:24.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2228 2024-09-18T00:43:24.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:25.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:25.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:43:25.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:25.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:25.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:43:25.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:43:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:43:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:43:25.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:43:25.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2230 2024-09-18T00:43:25.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2229 2024-09-18T00:43:25.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:25.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:26.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:26.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:43:26.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:26.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:26.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:43:26.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:43:26.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2231 2024-09-18T00:43:26.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:26.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:43:26.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:26.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:26.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:43:26.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:43:26.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:26.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2232 2024-09-18T00:43:26.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:27.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2234 2024-09-18T00:43:27.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2233 2024-09-18T00:43:27.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:27.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:27.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:27.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:43:27.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:27.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:27.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:43:27.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:43:27.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:27.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:43:27.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:27.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:27.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:43:27.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:43:28.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2235 2024-09-18T00:43:28.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2236 2024-09-18T00:43:28.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:28.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:28.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:43:28.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:28.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:28.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:43:28.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:43:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:43:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:43:28.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:43:28.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:29.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:29.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:43:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:43:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:43:29.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2238 2024-09-18T00:43:29.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:29.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:43:29.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:29.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:29.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:43:29.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:43:29.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:29.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2237 2024-09-18T00:43:29.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:43:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:43:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:43:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:43:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:43:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:43:30.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2239 2024-09-18T00:43:30.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2240 2024-09-18T00:43:30.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:30.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:31.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2242 2024-09-18T00:43:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:43:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:43:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:43:31.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2241 2024-09-18T00:43:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:43:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:43:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:43:31.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:31.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:32.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2244 2024-09-18T00:43:32.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2243 2024-09-18T00:43:32.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:32.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:32.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:43:32.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:43:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:43:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:43:32.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:43:33.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2245 2024-09-18T00:43:33.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:33.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2246 2024-09-18T00:43:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:43:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:43:33.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:43:33.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:33.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:33.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:43:33.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:33.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:33.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:43:33.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:43:34.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2247 2024-09-18T00:43:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:43:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:43:34.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:43:34.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:43:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:43:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:43:34.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2248 2024-09-18T00:43:34.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:35.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:35.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:43:35.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:35.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:35.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:43:35.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:43:35.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:35.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:43:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:43:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:43:35.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2249 2024-09-18T00:43:35.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2250 2024-09-18T00:43:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:35.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:43:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:43:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:43:36.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2251 2024-09-18T00:43:36.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:36.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:43:36.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:36.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:36.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:43:36.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:43:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:36.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2252 2024-09-18T00:43:36.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:37.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2254 2024-09-18T00:43:37.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2253 2024-09-18T00:43:37.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:37.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:37.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:43:37.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:37.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:37.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:43:37.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:43:37.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:37.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:43:37.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:37.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:37.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:43:37.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:43:38.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2255 2024-09-18T00:43:38.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2256 2024-09-18T00:43:38.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:38.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:38.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:38.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:43:38.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:38.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:38.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:43:38.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:43:38.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:38.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:43:38.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:38.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:38.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:43:38.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:43:39.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2258 2024-09-18T00:43:39.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2257 2024-09-18T00:43:39.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:39.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:39.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:39.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:43:39.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:39.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:39.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:43:39.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:43:39.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:43:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:43:39.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:43:40.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2259 2024-09-18T00:43:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:43:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:43:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:43:40.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:40.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2260 2024-09-18T00:43:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:43:40.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:40.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:40.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:43:40.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:43:40.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:41.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2261 2024-09-18T00:43:41.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2262 2024-09-18T00:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:43:41.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:41.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:43:41.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:41.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:41.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:43:41.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:43:41.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:41.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:42.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:42.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:43:42.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:42.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:42.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:43:42.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:43:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:43:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:43:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:43:42.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2264 2024-09-18T00:43:42.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2263 2024-09-18T00:43:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:42.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:42.823 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:43:42.818+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735ae340 2024-09-18T00:43:42.823 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:43:42.818+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735aedd0 2024-09-18T00:43:42.823 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:43:42.818+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d681c57ad0 2024-09-18T00:43:43.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:43.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:43:43.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:43.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:43.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:43:43.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:43:43.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2265 2024-09-18T00:43:43.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2266 2024-09-18T00:43:43.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:43.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:43:43.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:43.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:43.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:43:43.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:43:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:43.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:44.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:44.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:43:44.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:44.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:44.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:43:44.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:43:44.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:44.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:43:44.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:44.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:44.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:43:44.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:43:44.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2268 2024-09-18T00:43:44.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2267 2024-09-18T00:43:44.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:45.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:45.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:43:45.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:45.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:45.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:43:45.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:43:45.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:45.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:43:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:43:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:43:45.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2269 2024-09-18T00:43:45.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:45.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2270 2024-09-18T00:43:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:46.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2271 2024-09-18T00:43:46.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:46.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:43:46.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:46.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:46.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:43:46.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:43:46.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2272 2024-09-18T00:43:46.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:46.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:43:46.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:46.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:46.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:43:46.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:43:46.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:46.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:47.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2273 2024-09-18T00:43:47.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2274 2024-09-18T00:43:47.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:47.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:47.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:47.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:43:47.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:47.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:47.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:43:47.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:43:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:43:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:43:47.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:43:48.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:48.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:43:48.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:48.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:48.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:43:48.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:43:48.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2275 2024-09-18T00:43:48.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2276 2024-09-18T00:43:48.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:48.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:43:48.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:48.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:48.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:43:48.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:43:48.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:48.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:49.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2278 2024-09-18T00:43:49.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2277 2024-09-18T00:43:49.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:49.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:43:49.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:49.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:49.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:43:49.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:43:49.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:43:49.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 97575 2024-09-18T00:43:49.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:49.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:50.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:50.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:43:50.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:50.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:43:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:43:50.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:50.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:43:50.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:50.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:50.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:43:50.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:43:50.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2280 2024-09-18T00:43:50.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2279 2024-09-18T00:43:50.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:50.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2281 2024-09-18T00:43:51.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:43:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:43:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:43:51.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:43:51.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 97738 2024-09-18T00:43:52.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2282 2024-09-18T00:43:52.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2283 2024-09-18T00:43:52.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:52.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:52.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:52.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:43:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:43:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:43:52.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:52.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:43:52.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:52.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:52.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:43:52.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:43:53.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2284 2024-09-18T00:43:53.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:53.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:43:53.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:53.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:53.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:43:53.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:43:53.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:43:53.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 97891 2024-09-18T00:43:53.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:43:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2285 2024-09-18T00:43:54.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:54.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2286 2024-09-18T00:43:54.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:55.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:43:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:43:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:43:55.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:43:55.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 98047 2024-09-18T00:43:55.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2287 2024-09-18T00:43:55.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:56.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2288 2024-09-18T00:43:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:43:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:43:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:43:56.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:57.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2289 2024-09-18T00:43:57.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:57.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:43:57.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 98242 2024-09-18T00:43:57.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:57.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:43:57.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:57.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:57.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:43:57.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:43:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:43:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:43:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:43:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:43:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:43:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:43:58.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2290 2024-09-18T00:43:58.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:43:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 98415 2024-09-18T00:43:59.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2291 2024-09-18T00:43:59.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:43:59.916 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:43:59.914+0000 7f849c00d640 -1 reset not still connected to 0x5629b8c0bee0 2024-09-18T00:43:59.916 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:43:59.914+0000 7f849c00d640 -1 reset not still connected to 0x5629bbcc9ba0 2024-09-18T00:43:59.916 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:43:59.914+0000 7f849c00d640 -1 reset not still connected to 0x5629d36244e0 2024-09-18T00:43:59.916 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:43:59.914+0000 7f849c00d640 -1 reset not still connected to 0x5629e3328c30 2024-09-18T00:44:00.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:44:00.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:44:00.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:44:00.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:44:00.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:44:00.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:44:00.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2292 2024-09-18T00:44:00.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:44:01.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:44:01.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 98571 2024-09-18T00:44:02.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2293 2024-09-18T00:44:02.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:44:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:44:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:44:02.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:44:02.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:44:02.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:44:02.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:44:03.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:44:03.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 98886 2024-09-18T00:44:04.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2294 2024-09-18T00:44:04.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:44:05.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:44:05.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 99045 2024-09-18T00:44:07.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:44:07.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:44:07.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 97443 2024-09-18T00:46:22.813 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:46:22.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:46:22.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:46:22.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 97576 2024-09-18T00:46:23.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:46:23.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:46:23.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:46:23.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 97739 2024-09-18T00:46:24.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:46:24.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:46:24.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:46:24.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 97892 2024-09-18T00:46:26.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:46:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:46:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:46:26.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 98048 2024-09-18T00:46:28.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:46:28.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:46:28.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:46:28.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 98243 2024-09-18T00:46:31.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:46:31.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:46:31.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:46:31.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 98416 2024-09-18T00:46:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:46:33.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:46:33.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:46:33.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 98572 2024-09-18T00:46:34.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:46:34.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:46:34.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:46:34.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 98887 2024-09-18T00:46:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:46:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:46:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:46:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 99049 2024-09-18T00:46:39.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:46:39.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:46:39.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:46:39.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:46:39.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:46:39.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:46:39.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 1 2024-09-18T00:46:39.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:46:39.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:46:39.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd0 2024-09-18T00:46:39.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary1 2024-09-18T00:46:39.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 2 2024-09-18T00:46:39.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary1 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary1 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd1 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary2 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary2 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary2 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 3 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 4 2024-09-18T00:46:39.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:46:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd2 2024-09-18T00:46:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary3 2024-09-18T00:46:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary3 2024-09-18T00:46:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:46:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd3 2024-09-18T00:46:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary3 2024-09-18T00:46:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary4 2024-09-18T00:46:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary4 2024-09-18T00:46:39.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:46:39.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary4 2024-09-18T00:46:39.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:46:39.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:46:39.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 5 2024-09-18T00:46:39.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:46:39.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:46:39.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 6 2024-09-18T00:46:39.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd4 2024-09-18T00:46:39.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary5 2024-09-18T00:46:39.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary5 2024-09-18T00:46:39.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd5 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary5 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary6 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary6 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary6 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 7 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:46:39.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd6 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary7 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary7 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary7 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 104897 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 10 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd9 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary10 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary10 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary10 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 9 2024-09-18T00:46:39.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd8 2024-09-18T00:46:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary9 2024-09-18T00:46:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary9 2024-09-18T00:46:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:46:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary9 2024-09-18T00:46:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 8 2024-09-18T00:46:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd7 2024-09-18T00:46:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary8 2024-09-18T00:46:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary8 2024-09-18T00:46:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:46:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary8 2024-09-18T00:46:39.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:46:39.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:46:39.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:46:39.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:46:39.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:46:39.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:46:39.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:46:39.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:46:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:46:39.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:46:39.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:46:39.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:46:39.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:46:39.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:46:39.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:46:39.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:46:39.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:46:39.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:46:39.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:46:39.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:47:34.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=931aafa3c115b0dda1e5abf534e87c71 2024-09-18T00:47:34.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary7 2024-09-18T00:47:37.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary7 2024-09-18T00:47:37.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:37.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:37.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:47:37.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary7 2024-09-18T00:47:38.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:47:38.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+unknown 2024-09-18T00:47:38.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:47:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:38.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:39.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:47:39.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:39.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:39.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:47:39.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:39.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:39.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:39.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:39.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:39.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:47:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:47:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: ab0d841f-81f9-4384-b3bd-341cb0d2c745 2024-09-18T00:47:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:39 2024-09-18T00:47:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:39' 2024-09-18T00:47:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:40.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:40.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:40.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:40.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:40.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:40.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:40.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:40.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:39' 2024-09-18T00:47:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+unknown '2024-09-18 00:47:39' 2024-09-18T00:47:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:47:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:39' 2024-09-18T00:47:40.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:40.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:40.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:40.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=f0513788f0dd1b98181dddecb2eb4a41 2024-09-18T00:47:40.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary3 2024-09-18T00:47:40.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":952320.0,"last_snapshot_bytes":1716224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620225,"remote_snapshot_timestamp":1726620225,"replay_state":"idle"}2024-09-18 00:47:39image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2170880.0,"last_snapshot_bytes":2629632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620243,"remote_snapshot_timestamp":1726620243,"replay_state":"idle"}2024-09-18 00:47:39image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":4261888.0,"last_snapshot_bytes":4669440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620227,"remote_snapshot_timestamp":1726620227,"replay_state":"idle"}2024-09-18 00:47:39image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2564096,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620229,"remote_snapshot_timestamp":1726620229,"replay_state":"idle"}2024-09-18 00:47:39image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2592768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620231,"remote_snapshot_timestamp":1726620231,"replay_state":"idle"}2024-09-18 00:47:39image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2156544.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620233,"remote_snapshot_timestamp":1726620233,"replay_state":"idle"}2024-09-18 00:47:39image-primary61595563f-88e0-4d3f-be52-370c261044d6up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620235,"remote_snapshot_timestamp":1726620235,"replay_state":"idle"}2024-09-18 00:47:39image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:39image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2095104.0,"last_snapshot_bytes":2584576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620239,"remote_snapshot_timestamp":1726620239,"replay_state":"idle"}2024-09-18 00:47:39image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2160640.0,"last_snapshot_bytes":2568192,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620241,"remote_snapshot_timestamp":1726620241,"replay_state":"idle"}2024-09-18 00:47:39' 2024-09-18T00:47:40.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:40.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:47:40.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=175d399444b8b369d624607c0cbfd529 2024-09-18T00:47:40.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary2 2024-09-18T00:47:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:39' 2024-09-18T00:47:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:47:40.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:40.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:40.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:40.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:40.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:40.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:40.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary7 up+unknown 2024-09-18T00:47:40.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:40.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:40.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:47:40.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:40.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:40.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:40.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:40.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=d47b9ab006323715112a113c8bae7f8c 2024-09-18T00:47:41.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary1 2024-09-18T00:47:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary3 2024-09-18T00:47:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:47:41.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary3 2024-09-18T00:47:41.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary2 2024-09-18T00:47:41.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:41.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:41.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:47:41.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary2 2024-09-18T00:47:42.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=9cbbff0bc865f5bfb49b143be63cf3ac 2024-09-18T00:47:42.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary5 2024-09-18T00:47:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary7 up+unknown '' '' 2024-09-18T00:47:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:47:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:42.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:42.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:42.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary7 2024-09-18T00:47:42.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary1 2024-09-18T00:47:42.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:42.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:42.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:47:42.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary1 2024-09-18T00:47:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:47:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: ab0d841f-81f9-4384-b3bd-341cb0d2c745 2024-09-18T00:47:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:39 2024-09-18T00:47:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:47:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:42.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:42.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:39' 2024-09-18T00:47:42.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:42.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:42.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:42.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:42.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:42.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:42.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:42.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:42.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:42.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:42.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:42.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:42.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:39' 2024-09-18T00:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary7 up+unknown '2024-09-18 00:47:39' 2024-09-18T00:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:39' 2024-09-18T00:47:42.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:42.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:42.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:42.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1942614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":952320.0,"last_snapshot_bytes":1716224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620225,"remote_snapshot_timestamp":1726620225,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:39image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2170880.0,"last_snapshot_bytes":2629632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620243,"remote_snapshot_timestamp":1726620243,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:39image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":4261888.0,"last_snapshot_bytes":4669440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620227,"remote_snapshot_timestamp":1726620227,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:39image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2564096,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620229,"remote_snapshot_timestamp":1726620229,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:39image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2592768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620231,"remote_snapshot_timestamp":1726620231,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:39image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2156544.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620233,"remote_snapshot_timestamp":1726620233,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:39image-primary61595563f-88e0-4d3f-be52-370c261044d6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620235,"remote_snapshot_timestamp":1726620235,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:39image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:39image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2095104.0,"last_snapshot_bytes":2584576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620239,"remote_snapshot_timestamp":1726620239,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:39image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2160640.0,"last_snapshot_bytes":2568192,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620241,"remote_snapshot_timestamp":1726620241,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:39' 2024-09-18T00:47:42.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:42.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:47:42.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:39' 2024-09-18T00:47:42.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:47:42.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=eec8f4ea6982c56c2291f07f87973fa6 2024-09-18T00:47:42.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary8 2024-09-18T00:47:42.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:42.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:42.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary7 2024-09-18T00:47:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:47:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:47:42.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary7 2024-09-18T00:47:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:47:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:47:42.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:47:43.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=a542f244bace06a2446ce83cfffea3aa 2024-09-18T00:47:43.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary4 2024-09-18T00:47:43.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary5 2024-09-18T00:47:43.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:43.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:43.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:47:43.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary5 2024-09-18T00:47:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=14fa8ca9f7323978560095a0f007800f 2024-09-18T00:47:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary9 2024-09-18T00:47:43.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:47:43.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:47:43.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+unknown 2024-09-18T00:47:43.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:43.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:43.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:47:43.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:43.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:43.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:43.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:43.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:43.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:47:43.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+unknown 2024-09-18T00:47:43.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:43.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:43.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:47:43.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:43.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:43.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:43.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:43.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+unknown 2024-09-18T00:47:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:47:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:43.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:43.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd0 2024-09-18T00:47:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:47:43.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary8 2024-09-18T00:47:43.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:47:43.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:43.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:43.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:47:43.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary8 2024-09-18T00:47:43.900 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:47:43.900+0000 7f52d7c6c640 -1 reset not still connected to 0x557872bb1380 2024-09-18T00:47:44.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary4 2024-09-18T00:47:44.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:44.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:44.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:47:44.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary4 2024-09-18T00:47:44.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=0b09dae3202a0b5cfce525bf71e50115 2024-09-18T00:47:44.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary10 2024-09-18T00:47:44.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=0677ccfa03d72074c97b6cace1f0bbdd 2024-09-18T00:47:44.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary6 2024-09-18T00:47:44.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:47:44.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:44.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:44.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:47:44.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:44.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:44.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:44.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:44.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:44.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:47:44.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:47:44.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:44.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:44.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:47:44.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:44.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:44.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:44.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:44.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:44.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:47:44.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary9 2024-09-18T00:47:44.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:44.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:44.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:47:44.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary9 2024-09-18T00:47:44.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T00:47:44.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:44.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:44.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:47:44.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:44.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:44.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:44.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:44.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:44.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:47:44.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:47:44.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70 2024-09-18T00:47:44.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:44.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:44.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:44.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:44.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:44.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:44.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:47:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":4261888.0,"last_snapshot_bytes":4669440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620227,"remote_snapshot_timestamp":1726620227,"replay_state":"idle"} 2024-09-18T00:47:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:39' 2024-09-18T00:47:44.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:44.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:44.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:44.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:44.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:44.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:44.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:44.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:44.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:44.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:44' 2024-09-18T00:47:44.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+unknown '2024-09-18 00:47:44' 2024-09-18T00:47:44.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:44.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:44.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:47:44.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:44.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:44' 2024-09-18T00:47:44.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:44.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:44.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:47:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 1538ca69-026d-4504-986b-6ca3056a191b 2024-09-18T00:47:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:44.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:47:44.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":952320.0,"last_snapshot_bytes":1716224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620225,"remote_snapshot_timestamp":1726620225,"replay_state":"idle"} 2024-09-18T00:47:44.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:39' 2024-09-18T00:47:44.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:44.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:44.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:44.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:44.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:44.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:44.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:44.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:44.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:44.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:44.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:44' 2024-09-18T00:47:44.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+unknown '2024-09-18 00:47:44' 2024-09-18T00:47:44.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:44.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:44.670 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:47:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:44' 2024-09-18T00:47:44.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:44.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:44.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:47:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 2cd3b2a7-cd9c-456d-b051-1ce989af2010 2024-09-18T00:47:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:44.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:44.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:44.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:44.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:47:44.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2564096,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620229,"remote_snapshot_timestamp":1726620229,"replay_state":"idle"} 2024-09-18T00:47:44.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:39' 2024-09-18T00:47:44.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:44.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:44.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:44.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:44.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:44.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:44.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:44.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:44.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:44.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:44.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:44.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:44' 2024-09-18T00:47:44.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+unknown '2024-09-18 00:47:44' 2024-09-18T00:47:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:47:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:44' 2024-09-18T00:47:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:44.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":952320.0,"last_snapshot_bytes":1716224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620225,"remote_snapshot_timestamp":1726620225,"replay_state":"idle"}2024-09-18 00:47:39image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2170880.0,"last_snapshot_bytes":2629632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620243,"remote_snapshot_timestamp":1726620243,"replay_state":"idle"}2024-09-18 00:47:39image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":4261888.0,"last_snapshot_bytes":4669440,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620227,"remote_snapshot_timestamp":1726620227,"replay_state":"idle"}2024-09-18 00:47:39image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2564096,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620229,"remote_snapshot_timestamp":1726620229,"replay_state":"idle"}2024-09-18 00:47:39image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2592768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620231,"remote_snapshot_timestamp":1726620231,"replay_state":"idle"}2024-09-18 00:47:39image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2156544.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620233,"remote_snapshot_timestamp":1726620233,"replay_state":"idle"}2024-09-18 00:47:39image-primary61595563f-88e0-4d3f-be52-370c261044d6up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620235,"remote_snapshot_timestamp":1726620235,"replay_state":"idle"}2024-09-18 00:47:39image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2095104.0,"last_snapshot_bytes":2584576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620239,"remote_snapshot_timestamp":1726620239,"replay_state":"idle"}2024-09-18 00:47:39image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2160640.0,"last_snapshot_bytes":2568192,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620241,"remote_snapshot_timestamp":1726620241,"replay_state":"idle"}2024-09-18 00:47:39' 2024-09-18T00:47:44.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:44.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:47:44.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:44' 2024-09-18T00:47:44.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:47:44.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:44.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:44.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:44.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:44.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:44.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:44.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary2 up+unknown 2024-09-18T00:47:44.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:44.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:44.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:47:44.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:44.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:44.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:44.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:44.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:44.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary6 2024-09-18T00:47:44.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:44.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:44.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:47:44.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary6 2024-09-18T00:47:44.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary10 2024-09-18T00:47:44.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:44.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:44.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:47:44.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary10 2024-09-18T00:47:44.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":952320.0,"last_snapshot_bytes":1716224,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620225,"remote_snapshot_timestamp":1726620225,"replay_state":"idle"}2024-09-18 00:47:39image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2170880.0,"last_snapshot_bytes":2629632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620243,"remote_snapshot_timestamp":1726620243,"replay_state":"idle"}2024-09-18 00:47:39image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:47:44image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 00:47:44image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2592768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620231,"remote_snapshot_timestamp":1726620231,"replay_state":"idle"}2024-09-18 00:47:39image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2156544.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620233,"remote_snapshot_timestamp":1726620233,"replay_state":"idle"}2024-09-18 00:47:39image-primary61595563f-88e0-4d3f-be52-370c261044d6up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620235,"remote_snapshot_timestamp":1726620235,"replay_state":"idle"}2024-09-18 00:47:39image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2095104.0,"last_snapshot_bytes":2584576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620239,"remote_snapshot_timestamp":1726620239,"replay_state":"idle"}2024-09-18 00:47:39image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2160640.0,"last_snapshot_bytes":2568192,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620241,"remote_snapshot_timestamp":1726620241,"replay_state":"idle"}2024-09-18 00:47:39' 2024-09-18T00:47:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:47:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:44' 2024-09-18T00:47:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:47:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:44.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary1 up+unknown 2024-09-18T00:47:44.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:44.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:44.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:47:44.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:44.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:44.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:44.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:44.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:45.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 00:47:44image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2170880.0,"last_snapshot_bytes":2629632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620243,"remote_snapshot_timestamp":1726620243,"replay_state":"idle"}2024-09-18 00:47:39image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 00:47:44image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 00:47:44image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2592768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620231,"remote_snapshot_timestamp":1726620231,"replay_state":"idle"}2024-09-18 00:47:39image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2156544.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620233,"remote_snapshot_timestamp":1726620233,"replay_state":"idle"}2024-09-18 00:47:39image-primary61595563f-88e0-4d3f-be52-370c261044d6up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620235,"remote_snapshot_timestamp":1726620235,"replay_state":"idle"}2024-09-18 00:47:39image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2095104.0,"last_snapshot_bytes":2584576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620239,"remote_snapshot_timestamp":1726620239,"replay_state":"idle"}2024-09-18 00:47:39image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2160640.0,"last_snapshot_bytes":2568192,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620241,"remote_snapshot_timestamp":1726620241,"replay_state":"idle"}2024-09-18 00:47:39' 2024-09-18T00:47:45.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:45.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:47:45.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:44' 2024-09-18T00:47:45.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:47:45.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:45.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:45.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:45.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:45.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary3 up+unknown 2024-09-18T00:47:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:47:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:45.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:45.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:47:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:47:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:47:45.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+unknown 2024-09-18T00:47:45.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:45.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:45.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:47:45.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:45.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:45.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:45.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:45.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:45.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+unknown 2024-09-18T00:47:45.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:45.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:45.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:47:45.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:45.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:45.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:45.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:45.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:45.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+unknown 2024-09-18T00:47:45.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:45.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:45.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:47:45.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:45.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:45.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:45.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:45.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:45.578 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:47:45.576+0000 7f849c00d640 -1 reset not still connected to 0x5629e3328c30 2024-09-18T00:47:45.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary2 up+unknown '' '' 2024-09-18T00:47:45.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:45.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:45.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:47:45.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:45.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary2 2024-09-18T00:47:45.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:47:45.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70 2024-09-18T00:47:45.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:45.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:47:45.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:45.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:45.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:45.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:47:45.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:45.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:45.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44' 2024-09-18T00:47:45.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:45.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:45.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:45.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:47:45.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:45.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:45.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:45.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:45.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:45.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:44' 2024-09-18T00:47:45.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary2 up+unknown '2024-09-18 00:47:44' 2024-09-18T00:47:45.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:45.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:45.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:47:45.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:45.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:44' 2024-09-18T00:47:45.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:45.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:45.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:45.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary1 up+unknown '' '' 2024-09-18T00:47:45.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:45.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:45.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:47:45.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:45.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:45.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:45.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:45.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:45.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary1 2024-09-18T00:47:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:47:45cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:44image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2170880.0,"last_snapshot_bytes":2629632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620243,"remote_snapshot_timestamp":1726620243,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:47:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:44image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:47:45cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:44image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2592768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620231,"remote_snapshot_timestamp":1726620231,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2156544.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620233,"remote_snapshot_timestamp":1726620233,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary61595563f-88e0-4d3f-be52-370c261044d6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620235,"remote_snapshot_timestamp":1726620235,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}2024-09-18 00:47:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2095104.0,"last_snapshot_bytes":2584576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620239,"remote_snapshot_timestamp":1726620239,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2160640.0,"last_snapshot_bytes":2568192,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620241,"remote_snapshot_timestamp":1726620241,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44' 2024-09-18T00:47:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:46.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:47:46.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:44' 2024-09-18T00:47:46.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:47:46.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:47:46.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 1538ca69-026d-4504-986b-6ca3056a191b 2024-09-18T00:47:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:47:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:45 2024-09-18T00:47:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:47:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44' 2024-09-18T00:47:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:46.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:46.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:46.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:46.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:47:46.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:46.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary2 2024-09-18T00:47:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:47:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:47:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary2 2024-09-18T00:47:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:45' 2024-09-18T00:47:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary1 up+unknown '2024-09-18 00:47:45' 2024-09-18T00:47:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:47:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:45' 2024-09-18T00:47:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:46.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:46.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary3 up+unknown '' '' 2024-09-18T00:47:46.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:46.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:46.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:47:46.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:46.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:46.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:46.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:46.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:46.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary3 2024-09-18T00:47:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:47:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 2cd3b2a7-cd9c-456d-b051-1ce989af2010 2024-09-18T00:47:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:47:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:45 2024-09-18T00:47:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:47:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:46.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:46.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44' 2024-09-18T00:47:46.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:46.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:46.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:46.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T00:47:46.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:46.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:46.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:46.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:46.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:46.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:46.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:46.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:45' 2024-09-18T00:47:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary3 up+unknown '2024-09-18 00:47:45' 2024-09-18T00:47:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:47:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:46.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:45' 2024-09-18T00:47:46.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:46.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:46.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:46.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:47:45cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:44image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2170880.0,"last_snapshot_bytes":2629632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620243,"remote_snapshot_timestamp":1726620243,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:47:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:44image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:47:45cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:44image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2592768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620231,"remote_snapshot_timestamp":1726620231,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2156544.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620233,"remote_snapshot_timestamp":1726620233,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary61595563f-88e0-4d3f-be52-370c261044d6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620235,"remote_snapshot_timestamp":1726620235,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}2024-09-18 00:47:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2095104.0,"last_snapshot_bytes":2584576,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620239,"remote_snapshot_timestamp":1726620239,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2160640.0,"last_snapshot_bytes":2568192,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620241,"remote_snapshot_timestamp":1726620241,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44' 2024-09-18T00:47:46.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:46.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:47:46.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:45' 2024-09-18T00:47:46.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:47:46.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:46.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:46.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:46.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:46.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:46.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:46.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary1 2024-09-18T00:47:46.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:46.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:46.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:47:46.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:47:46.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary1 2024-09-18T00:47:46.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:47:46.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:47:46.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:47:46.517 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:47:46.516+0000 7fd4c158b640 -1 reset not still connected to 0x5643663b75f0 2024-09-18T00:47:46.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+unknown 2024-09-18T00:47:46.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:46.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:46.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:47:46.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:46.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:46.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:46.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:46.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:46.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+unknown 2024-09-18T00:47:46.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:46.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:46.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:47:46.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:46.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:46.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:46.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:46.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:47:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:47:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:46.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:46.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:47:46.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+unknown 2024-09-18T00:47:46.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:46.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:46.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:47:46.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:46.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:46.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:46.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:46.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:46.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:47:46.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:46.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:46.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:47:46.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:46.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:46.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:46.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:46.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:46.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:47:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:47:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:47:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:46.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:46.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:47:46.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:47:46.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:47:46.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4dabe928-cfdd-47d0-97f0-a3be00b4b2a0 2024-09-18T00:47:46.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:47:46.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:47:46.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:46.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:46.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:46.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:46.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopping_replay 2024-09-18T00:47:46.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:47:46.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:46' 2024-09-18T00:47:46.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:46.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:47:46.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:46.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:47:46.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:47:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:47:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4ddc12ed-652e-41cb-a869-7273142e4cee 2024-09-18T00:47:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:47:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:47:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:46.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:46.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:46.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:47:46.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:46' 2024-09-18T00:47:46.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:46.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:47:46.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:46.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:47:46.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:47:46.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:47:45cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:44image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2170880.0,"last_snapshot_bytes":2629632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620243,"remote_snapshot_timestamp":1726620243,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:47:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:44image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:47:45cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:44image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+stopping_replayremote image demoted42614264mirror.0smithi0492024-09-18 00:47:46cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+stopping_replayremote image demoted42614264mirror.0smithi0492024-09-18 00:47:46cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary61595563f-88e0-4d3f-be52-370c261044d6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620235,"remote_snapshot_timestamp":1726620235,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}2024-09-18 00:47:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:47:46cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2160640.0,"last_snapshot_bytes":2568192,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620241,"remote_snapshot_timestamp":1726620241,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:47:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:47:44' 2024-09-18T00:47:46.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:47:46.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:47:46.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8f3bd77d-71f5-46c9-8ccc-36529855a3b4 2024-09-18T00:47:46.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:47:46.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:47:46.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:46.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:46.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:46.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:46.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:46.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:47:46.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:46' 2024-09-18T00:47:46.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:46.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:47:46.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:47:46.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:46.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:47:46.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:45' 2024-09-18T00:47:46.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:47:46.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:46.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:46.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:46.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:46.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:46.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:46.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary3 2024-09-18T00:47:46.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:46.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:46.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:47:46.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:47:46.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary3 2024-09-18T00:47:46.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:47:46.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:47:47.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd1 2024-09-18T00:47:47.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:47:47.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:47:47.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd2 2024-09-18T00:47:47.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:47:47.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:47:47.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T00:47:47.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:47.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:47.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:47:47.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:47.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:47.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:47.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:47.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:47:47.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T00:47:47.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:47:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:47:47.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T00:47:47.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:47:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:47.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:47:47.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:47:47.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 47cdb799-ea3d-491b-a019-2a4da9cb8764 2024-09-18T00:47:47.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:47:47.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:47:47.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:47.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:47.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:47.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:47.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:47:47.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2160640.0,"last_snapshot_bytes":2568192,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620241,"remote_snapshot_timestamp":1726620241,"replay_state":"idle"} 2024-09-18T00:47:47.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:39' 2024-09-18T00:47:47.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:47.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:47:47.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:47.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:47:47.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:47:47.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 1595563f-88e0-4d3f-be52-370c261044d6 2024-09-18T00:47:47.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:47:47.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:47:47.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:47.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:47.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:47.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:47.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:47:47.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2142208.0,"last_snapshot_bytes":2588672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620235,"remote_snapshot_timestamp":1726620235,"replay_state":"idle"} 2024-09-18T00:47:47.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:39' 2024-09-18T00:47:47.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:47.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:47:47.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:47.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:47:47.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:47:47.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9b75893a-9ef9-468b-ad0c-88ee663b581a 2024-09-18T00:47:47.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:47:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:47:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:47:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2170880.0,"last_snapshot_bytes":2629632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620243,"remote_snapshot_timestamp":1726620243,"replay_state":"idle"} 2024-09-18T00:47:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:39' 2024-09-18T00:47:47.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:47.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:47:47.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:47:47.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:47:47.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:47:47.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:47:48.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd3 2024-09-18T00:47:48.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:47:48.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:47:48.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:47:48.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:48.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:48.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:47:48.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:48.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:48.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:48.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:48.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:48.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:47:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:47:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:47:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:48.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:48.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:47:48.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:47:48.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:48.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:48.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:47:48.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:48.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:48.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:48.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:48.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:48.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:47:48.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:47:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4dabe928-cfdd-47d0-97f0-a3be00b4b2a0 2024-09-18T00:47:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:47:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:47:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:47:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:46' 2024-09-18T00:47:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:47:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:47:48.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:47:48.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4ddc12ed-652e-41cb-a869-7273142e4cee 2024-09-18T00:47:48.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:47:48.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:47:48.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:48.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:48.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:48.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:48.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:48.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:47:48.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:46' 2024-09-18T00:47:48.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:48.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:47:48.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:48.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:47:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:47:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8f3bd77d-71f5-46c9-8ccc-36529855a3b4 2024-09-18T00:47:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:47:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:47:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:44 2024-09-18T00:47:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:47:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:46' 2024-09-18T00:47:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:47:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:47:49.262 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:47:49.260+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b040 2024-09-18T00:47:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T00:47:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:47:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:49.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:49.644 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:47:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T00:47:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:47:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:49.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:49.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:49.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:47:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T00:47:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:49.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:49.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:47:49.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:49.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:49.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:49.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:49.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:49.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:47:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:47:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 47cdb799-ea3d-491b-a019-2a4da9cb8764 2024-09-18T00:47:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49 2024-09-18T00:47:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49' 2024-09-18T00:47:49.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:49.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:49.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:49.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:49.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:49.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:49.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:49.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:49.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:49.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:49.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:49.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:49.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:49.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+unknown '2024-09-18 00:47:49' 2024-09-18T00:47:49.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:49.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:49.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:47:49.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:49.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:49' 2024-09-18T00:47:49.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:49.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:49.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:50.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:47:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9b75893a-9ef9-468b-ad0c-88ee663b581a 2024-09-18T00:47:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49 2024-09-18T00:47:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49' 2024-09-18T00:47:50.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:50.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:50.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:50.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:50.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:50.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:50.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:50.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:50.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:47:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 1595563f-88e0-4d3f-be52-370c261044d6 2024-09-18T00:47:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49 2024-09-18T00:47:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49' 2024-09-18T00:47:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:50.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:50.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:50.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:50.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+unknown '2024-09-18 00:47:49' 2024-09-18T00:47:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:47:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:49' 2024-09-18T00:47:50.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:50.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:50.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:50.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:50.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:50.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:50.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:50.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:50.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:50.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:50.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:50.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:50.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:50.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:50.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:50.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+unknown '2024-09-18 00:47:49' 2024-09-18T00:47:50.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:50.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:50.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:47:50.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:50.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:49' 2024-09-18T00:47:50.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:50.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:50.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:50.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary61595563f-88e0-4d3f-be52-370c261044d6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49' 2024-09-18T00:47:50.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:50.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:47:50.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:50.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:47:50.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:50.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:50.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:50.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:50.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:50.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:50.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary9 up+unknown 2024-09-18T00:47:50.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:50.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:50.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:47:50.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:50.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:50.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:50.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:50.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:50.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary61595563f-88e0-4d3f-be52-370c261044d6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49' 2024-09-18T00:47:50.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:50.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:47:50.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:50.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:47:50.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:50.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:50.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:50.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary10 up+unknown 2024-09-18T00:47:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:47:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:50.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary61595563f-88e0-4d3f-be52-370c261044d6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49' 2024-09-18T00:47:50.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:50.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:47:50.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:50.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:47:50.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:50.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:50.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:50.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:50.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:50.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:50.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary6 up+unknown 2024-09-18T00:47:50.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:50.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:50.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:47:50.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:50.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:50.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:50.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:50.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:51.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary9 up+unknown '' '' 2024-09-18T00:47:51.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:51.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:51.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:47:51.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:51.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:51.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:51.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:51.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:51.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary9 2024-09-18T00:47:51.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary10 up+unknown '' '' 2024-09-18T00:47:51.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:51.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:51.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:47:51.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:51.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:51.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:51.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:51.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:51.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary10 2024-09-18T00:47:51.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary6 up+unknown '' '' 2024-09-18T00:47:51.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:51.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:51.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:47:51.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:51.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:51.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:51.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:51.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:51.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary6 2024-09-18T00:47:51.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:47:51.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9b75893a-9ef9-468b-ad0c-88ee663b581a 2024-09-18T00:47:51.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:51.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:51.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:51.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49 2024-09-18T00:47:51.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:51.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:47:51.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:51.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:51.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49' 2024-09-18T00:47:51.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:51.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:51.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:51.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:51.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:51.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:47:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 1595563f-88e0-4d3f-be52-370c261044d6 2024-09-18T00:47:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49 2024-09-18T00:47:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:47:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49' 2024-09-18T00:47:51.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:51.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:51.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:51.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:51.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:51.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:51.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:51.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:51.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:51.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:51.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:51.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:51.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:51.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:51.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary10 up+unknown '2024-09-18 00:47:49' 2024-09-18T00:47:51.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:51.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:51.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:47:51.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:51.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:49' 2024-09-18T00:47:51.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:51.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:51.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:51.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:51.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:51.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:51.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:51.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:51.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary6 up+unknown '2024-09-18 00:47:49' 2024-09-18T00:47:51.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:51.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:51.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:47:51.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:51.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:49' 2024-09-18T00:47:51.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:51.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:51.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:51.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:47:51.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 47cdb799-ea3d-491b-a019-2a4da9cb8764 2024-09-18T00:47:51.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:51.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:51.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:51.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49 2024-09-18T00:47:51.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:51.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:47:51.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:51.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:51.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49' 2024-09-18T00:47:51.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:51.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:51.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:51.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:51.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:51.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:51.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:51.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:51.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:51.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:51.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:51.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:51.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:51.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary9 up+unknown '2024-09-18 00:47:49' 2024-09-18T00:47:51.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:51.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:51.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:47:51.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:51.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:49' 2024-09-18T00:47:51.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:51.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:51.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:51.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}2024-09-18 00:47:49image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}2024-09-18 00:47:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}2024-09-18 00:47:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary61595563f-88e0-4d3f-be52-370c261044d6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}2024-09-18 00:47:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49' 2024-09-18T00:47:51.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:51.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:47:51.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:51.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:47:51.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:51.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:51.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:51.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary6 2024-09-18T00:47:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:47:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:47:51.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary6 2024-09-18T00:47:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}2024-09-18 00:47:49image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}2024-09-18 00:47:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}2024-09-18 00:47:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary61595563f-88e0-4d3f-be52-370c261044d6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}2024-09-18 00:47:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49' 2024-09-18T00:47:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:51.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:47:51.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:51.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:47:51.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:51.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:51.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:51.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary10 2024-09-18T00:47:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:47:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:47:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary10 2024-09-18T00:47:51.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}2024-09-18 00:47:49image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}2024-09-18 00:47:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}2024-09-18 00:47:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary61595563f-88e0-4d3f-be52-370c261044d6up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}2024-09-18 00:47:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:49' 2024-09-18T00:47:51.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:51.848 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:47:51.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:51.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:47:51.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:51.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:51.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:51.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:51.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary9 2024-09-18T00:47:51.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:51.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:51.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:47:51.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:47:51.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary9 2024-09-18T00:47:52.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:47:52.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:52.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:52.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:47:52.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:52.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:52.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:52.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:52.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:52.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:47:52.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:47:52.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:52.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:52.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:47:52.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:52.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:52.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:52.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:52.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:52.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:47:52.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:47:52.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:52.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:52.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:47:52.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:52.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:52.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:52.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:52.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:52.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:47:52.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:47:52.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:47:52.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:47:52.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:47:52.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:47:52.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:47:52.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4ddc12ed-652e-41cb-a869-7273142e4cee 2024-09-18T00:47:52.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:52.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:52.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:52.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49 2024-09-18T00:47:52.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:52.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:52.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:52.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:52.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49' 2024-09-18T00:47:52.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:52.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:52.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:52.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:52.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:52.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:52.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:52.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:52.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:52.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:52.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:52.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:47:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4dabe928-cfdd-47d0-97f0-a3be00b4b2a0 2024-09-18T00:47:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49 2024-09-18T00:47:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49' 2024-09-18T00:47:52.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:52.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:52.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:52.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:52.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+unknown '2024-09-18 00:47:49' 2024-09-18T00:47:52.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:47:52.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:52.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:52.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:47:52.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:52.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:49' 2024-09-18T00:47:52.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:52.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:47:52.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:52.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:52.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:52.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:52.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:52.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:47:52.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:52.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:47:52.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:52.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:52.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:52.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:52.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:52.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:52.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+unknown '2024-09-18 00:47:49' 2024-09-18T00:47:52.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:52.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:52.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:47:52.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:52.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:49' 2024-09-18T00:47:52.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:52.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:52.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:52.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:47:52.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8f3bd77d-71f5-46c9-8ccc-36529855a3b4 2024-09-18T00:47:52.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:52.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:52.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:52.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49 2024-09-18T00:47:52.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:52.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:47:52.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:52.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:52.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:49' 2024-09-18T00:47:52.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:52.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:52.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:52.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:52.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:52.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:52.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:52.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:52.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:52.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:52.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:52.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:52.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:52.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+unknown '2024-09-18 00:47:49' 2024-09-18T00:47:52.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:47:52.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:52.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:47:52.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:52.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:49' 2024-09-18T00:47:52.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:52.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:52.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:53.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd4 2024-09-18T00:47:53.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:47:53.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:47:53.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary61595563f-88e0-4d3f-be52-370c261044d6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49' 2024-09-18T00:47:53.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:53.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:47:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:53.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:53.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:53.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary5 up+unknown 2024-09-18T00:47:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:47:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:53.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:53.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary61595563f-88e0-4d3f-be52-370c261044d6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49' 2024-09-18T00:47:53.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:53.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:47:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:47:53.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:53.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:53.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:53.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:53.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:53.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary8 up+unknown 2024-09-18T00:47:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:47:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:53.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd5 2024-09-18T00:47:53.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:47:53.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING6442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary61595563f-88e0-4d3f-be52-370c261044d6up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:47:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:47:49image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:47:49cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:47:49' 2024-09-18T00:47:53.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:53.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:47:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:49' 2024-09-18T00:47:53.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:47:53.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:47:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:53.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:53.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:53.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:53.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary4 up+unknown 2024-09-18T00:47:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:47:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:47:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:47:53.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd6 2024-09-18T00:47:53.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:47:53.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:47:54.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary5 up+unknown '' '' 2024-09-18T00:47:54.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:54.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:54.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:47:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary5 2024-09-18T00:47:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary8 up+unknown '' '' 2024-09-18T00:47:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:47:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:54.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary8 2024-09-18T00:47:54.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary4 up+unknown '' '' 2024-09-18T00:47:54.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:54.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:54.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:47:54.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:54.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:47:54.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:47:54.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:54.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:54.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary4 2024-09-18T00:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8f3bd77d-71f5-46c9-8ccc-36529855a3b4 2024-09-18T00:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:54 2024-09-18T00:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:54' 2024-09-18T00:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:54.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:54.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:54.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:54.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:47:54.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4dabe928-cfdd-47d0-97f0-a3be00b4b2a0 2024-09-18T00:47:54.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:54.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:54.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:54.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:54 2024-09-18T00:47:54.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:54.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:47:54.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:54.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:54' 2024-09-18T00:47:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:54.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:54.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:54.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:54.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:54.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:54.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:54.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:54.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:54.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:54.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:54.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:54.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:54.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:54.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:54' 2024-09-18T00:47:54.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary8 up+unknown '2024-09-18 00:47:54' 2024-09-18T00:47:54.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:54.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:54.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:47:54.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:54.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:54' 2024-09-18T00:47:54.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:54' 2024-09-18T00:47:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary5 up+unknown '2024-09-18 00:47:54' 2024-09-18T00:47:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:54.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:47:54.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:54.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:54' 2024-09-18T00:47:54.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:54.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:54.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:47:54.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4ddc12ed-652e-41cb-a869-7273142e4cee 2024-09-18T00:47:54.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:54.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:54.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:47:54.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:54 2024-09-18T00:47:54.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:47:54.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:47:54.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:47:54.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:47:54.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:47:54' 2024-09-18T00:47:54.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:47:54.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:47:54.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:47:54.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:47:54.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:47:54.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:54.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:47:54.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:54.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:47:54.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:47:54.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:47:54.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:47:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:54' 2024-09-18T00:47:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary4 up+unknown '2024-09-18 00:47:54' 2024-09-18T00:47:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:47:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:47:54.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:47:54' 2024-09-18T00:47:54.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:47:54.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:47:54.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:47:54.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}2024-09-18 00:47:49image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}2024-09-18 00:47:54image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}2024-09-18 00:47:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}2024-09-18 00:47:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:54image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:54image-primary61595563f-88e0-4d3f-be52-370c261044d6up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}2024-09-18 00:47:54image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}2024-09-18 00:47:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:54image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}2024-09-18 00:47:54' 2024-09-18T00:47:54.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:54.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:47:54.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:54' 2024-09-18T00:47:54.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:47:54.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:54.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}2024-09-18 00:47:49image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}2024-09-18 00:47:54image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}2024-09-18 00:47:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}2024-09-18 00:47:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:54image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:54image-primary61595563f-88e0-4d3f-be52-370c261044d6up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}2024-09-18 00:47:54image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}2024-09-18 00:47:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:54image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}2024-09-18 00:47:54' 2024-09-18T00:47:54.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:54.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary5 2024-09-18T00:47:54.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:47:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:47:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary5 2024-09-18T00:47:54.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:47:54.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:54' 2024-09-18T00:47:54.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}2024-09-18 00:47:49image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}2024-09-18 00:47:54image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}2024-09-18 00:47:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}2024-09-18 00:47:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:54image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:54image-primary61595563f-88e0-4d3f-be52-370c261044d6up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}2024-09-18 00:47:54image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}2024-09-18 00:47:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:47:54image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:47:54cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}2024-09-18 00:47:54' 2024-09-18T00:47:54.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:47:55.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:47:55.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:47:55.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:47:54' 2024-09-18T00:47:55.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:47:55.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:55.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:55.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary8 2024-09-18T00:47:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:47:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:47:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary8 2024-09-18T00:47:55.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:47:55.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:47:55.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:47:55.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:47:55.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:55.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:47:55.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary4 2024-09-18T00:47:55.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:47:55.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:47:55.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:47:55.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:47:55.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary4 2024-09-18T00:47:55.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:47:55.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:47:55.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:47:55.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:47:55.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:47:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:47:55.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:47:55.623 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:47:55.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:47:56.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd7 2024-09-18T00:47:56.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:47:56.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:47:56.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd8 2024-09-18T00:47:56.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:47:56.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:47:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd9 2024-09-18T00:47:56.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:47:56.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:49:14.300 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:49:14.297+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735aedd0 2024-09-18T00:49:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=931aafa3c115b0dda1e5abf534e87c71 2024-09-18T00:49:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd0 2024-09-18T00:49:23.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 931aafa3c115b0dda1e5abf534e87c71 != \9\3\1\a\a\f\a\3\c\1\1\5\b\0\d\d\a\1\e\5\a\b\f\5\3\4\e\8\7\c\7\1 ]] 2024-09-18T00:49:31.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=175d399444b8b369d624607c0cbfd529 2024-09-18T00:49:31.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd2 2024-09-18T00:49:32.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 175d399444b8b369d624607c0cbfd529 != \1\7\5\d\3\9\9\4\4\4\b\8\b\3\6\9\d\6\2\4\6\0\7\c\0\c\b\f\d\5\2\9 ]] 2024-09-18T00:49:33.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=d47b9ab006323715112a113c8bae7f8c 2024-09-18T00:49:33.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd1 2024-09-18T00:49:33.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=f0513788f0dd1b98181dddecb2eb4a41 2024-09-18T00:49:33.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd3 2024-09-18T00:49:34.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ d47b9ab006323715112a113c8bae7f8c != \d\4\7\b\9\a\b\0\0\6\3\2\3\7\1\5\1\1\2\a\1\1\3\c\8\b\a\e\7\f\8\c ]] 2024-09-18T00:49:34.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:49:34.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 104898 2024-09-18T00:49:34.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:49:34.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 104899 2024-09-18T00:49:34.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ f0513788f0dd1b98181dddecb2eb4a41 != \f\0\5\1\3\7\8\8\f\0\d\d\1\b\9\8\1\8\1\d\d\d\e\c\b\2\e\b\4\a\4\1 ]] 2024-09-18T00:49:34.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:49:34.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 104901 2024-09-18T00:49:38.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=14fa8ca9f7323978560095a0f007800f 2024-09-18T00:49:38.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd5 2024-09-18T00:49:39.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 14fa8ca9f7323978560095a0f007800f != \1\4\f\a\8\c\a\9\f\7\3\2\3\9\7\8\5\6\0\0\9\5\a\0\f\0\0\7\8\0\0\f ]] 2024-09-18T00:49:39.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=0b09dae3202a0b5cfce525bf71e50115 2024-09-18T00:49:39.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd4 2024-09-18T00:49:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=0677ccfa03d72074c97b6cace1f0bbdd 2024-09-18T00:49:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd6 2024-09-18T00:49:40.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=9cbbff0bc865f5bfb49b143be63cf3ac 2024-09-18T00:49:40.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd9 2024-09-18T00:49:41.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 0677ccfa03d72074c97b6cace1f0bbdd != \0\6\7\7\c\c\f\a\0\3\d\7\2\0\7\4\c\9\7\b\6\c\a\c\e\1\f\0\b\b\d\d ]] 2024-09-18T00:49:41.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 0b09dae3202a0b5cfce525bf71e50115 != \0\b\0\9\d\a\e\3\2\0\2\a\0\b\5\c\f\c\e\5\2\5\b\f\7\1\e\5\0\1\1\5 ]] 2024-09-18T00:49:41.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 9cbbff0bc865f5bfb49b143be63cf3ac != \9\c\b\b\f\f\0\b\c\8\6\5\f\5\b\f\b\4\9\b\1\4\3\b\e\6\3\c\f\3\a\c ]] 2024-09-18T00:49:41.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=a542f244bace06a2446ce83cfffea3aa 2024-09-18T00:49:41.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd7 2024-09-18T00:49:42.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=eec8f4ea6982c56c2291f07f87973fa6 2024-09-18T00:49:42.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd8 2024-09-18T00:49:42.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ a542f244bace06a2446ce83cfffea3aa != \a\5\4\2\f\2\4\4\b\a\c\e\0\6\a\2\4\4\6\c\e\8\3\c\f\f\f\e\a\3\a\a ]] 2024-09-18T00:49:42.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:49:42.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 104903 2024-09-18T00:49:42.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:49:42.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 104906 2024-09-18T00:49:42.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:49:42.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 104907 2024-09-18T00:49:42.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:49:42.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 104910 2024-09-18T00:49:42.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ eec8f4ea6982c56c2291f07f87973fa6 != \e\e\c\8\f\4\e\a\6\9\8\2\c\5\6\c\2\2\9\1\f\0\7\f\8\7\9\7\3\f\a\6 ]] 2024-09-18T00:49:42.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:49:42.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 104911 2024-09-18T00:49:42.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:49:42.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 104912 2024-09-18T00:49:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:49:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:49:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:49:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary1 2024-09-18T00:49:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:49:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:49:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+replaying 2024-09-18T00:49:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:42.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:49:42.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:42.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:49:42.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:49:42.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:49:42.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:49:43.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+replaying '' '' 2024-09-18T00:49:43.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:43.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:43.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:49:43.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:43.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:49:43.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:49:43.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:49:43.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:49:43.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:49:43.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:49:43.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 1538ca69-026d-4504-986b-6ca3056a191b 2024-09-18T00:49:43.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:49:43.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72761344.0,"last_snapshot_bytes":72761344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620466,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"} 2024-09-18T00:49:43.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:49:43.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:39 2024-09-18T00:49:43.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:49:43.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:49:43.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:49:43.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:49:43.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:39' 2024-09-18T00:49:43.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:49:43.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:49:43.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:49:43.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72761344.0,"last_snapshot_bytes":72761344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620466,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"} 2024-09-18T00:49:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:49:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:49:43.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:49:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:49:43.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:49:43.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:49:43.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:49:43.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:49:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:49:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+replaying '2024-09-18 00:49:39' 2024-09-18T00:49:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:49:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:49:39' 2024-09-18T00:49:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:49:43.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:49:43.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:49:44.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary11538ca69-026d-4504-986b-6ca3056a191bup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72761344.0,"last_snapshot_bytes":72761344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620466,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:39image-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:39image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":146468864.0,"last_snapshot_bytes":146468864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620466,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:39image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70193152.0,"last_snapshot_bytes":70193152,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620467,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:39image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70037504.0,"last_snapshot_bytes":70037504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:39image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:39image-primary61595563f-88e0-4d3f-be52-370c261044d6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70307840.0,"last_snapshot_bytes":70307840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:39image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620462,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:39image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69959680.0,"last_snapshot_bytes":69959680,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:39image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70164480.0,"last_snapshot_bytes":70164480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:39' 2024-09-18T00:49:44.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:49:44.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:49:45.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:49:45.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:49:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:49:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:49:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:49:45.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:49:45.518 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:45.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:45.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary1 2024-09-18T00:49:45.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:49:45.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:45.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:49:45.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary1 2024-09-18T00:49:45.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary1 2024-09-18T00:49:47.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:49:47.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary1 2024-09-18T00:49:47.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:47.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:47.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:49:47.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:49:47.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:49:47.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary1 2024-09-18T00:49:47.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:47.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary1 2024-09-18T00:49:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:49:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary2 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+replaying 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:49:47.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:49:47.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:49:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+replaying '' '' 2024-09-18T00:49:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:48.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:49:48.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:48.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:49:48.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:49:48.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:49:48.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:49:48.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:49:48.937 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:48.932+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:49:48.937 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:48.932+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:49:48.944 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:48.940+0000 7f849c00d640 -1 reset not still connected to 0x5629ae79fad0 2024-09-18T00:49:48.944 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:48.940+0000 7f849c00d640 -1 reset not still connected to 0x5629b5f785b0 2024-09-18T00:49:48.944 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:48.940+0000 7f849c00d640 -1 reset not still connected to 0x5629c3bd2f70 2024-09-18T00:49:48.944 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:48.940+0000 7f849c00d640 -1 reset not still connected to 0x5629c3bd3ba0 2024-09-18T00:49:48.945 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:48.940+0000 7f849c00d640 -1 reset not still connected to 0x5629ddabf450 2024-09-18T00:49:48.945 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:48.944+0000 7f849c00d640 -1 reset not still connected to 0x5629e3768c30 2024-09-18T00:49:48.945 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:48.944+0000 7f849c00d640 -1 reset not still connected to 0x5629e3768f70 2024-09-18T00:49:48.945 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:48.944+0000 7f849c00d640 -1 reset not still connected to 0x5629f9af75f0 2024-09-18T00:49:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:49:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70 2024-09-18T00:49:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:49:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":146468864.0,"last_snapshot_bytes":146468864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620466,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"} 2024-09-18T00:49:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:49:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:39 2024-09-18T00:49:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:49:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:49:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:49:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:49:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:44' 2024-09-18T00:49:49.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:49:49.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:49:49.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:49:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":146468864.0,"last_snapshot_bytes":146468864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620466,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"} 2024-09-18T00:49:49.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:49:49.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:49:49.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:49:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:49:49.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:49:49.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:49:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:49:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:49:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:49:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+replaying '2024-09-18 00:49:39' 2024-09-18T00:49:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:49.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:49:49.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:49.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:49:39' 2024-09-18T00:49:49.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:49:49.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:49:49.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:49:49.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:44image-primary2ea4bb5ad-f276-43a8-8faf-3ec1c28b0e70up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":146468864.0,"last_snapshot_bytes":146468864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620466,"remote_snapshot_timestamp":1726620466,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:44image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70193152.0,"last_snapshot_bytes":70193152,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620467,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:44image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70037504.0,"last_snapshot_bytes":70037504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:44image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:44image-primary61595563f-88e0-4d3f-be52-370c261044d6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70307840.0,"last_snapshot_bytes":70307840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:44image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620462,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:44image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69959680.0,"last_snapshot_bytes":69959680,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:44image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70164480.0,"last_snapshot_bytes":70164480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:44' 2024-09-18T00:49:49.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:49:49.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:49:49.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:49:49.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:49:49.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:49:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:49:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:49:49.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:49:49.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:49.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:49.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary2 2024-09-18T00:49:49.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:49:49.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:49.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:49:49.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary2 2024-09-18T00:49:49.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary2 2024-09-18T00:49:51.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:49:51.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary2 2024-09-18T00:49:51.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:51.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:51.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:49:51.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:49:51.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:49:51.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary2 2024-09-18T00:49:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:51.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary2 2024-09-18T00:49:51.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:49:51.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:49:51.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary3 2024-09-18T00:49:51.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:51.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:51.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:49:51.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:49:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+replaying 2024-09-18T00:49:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:49:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:49:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:49:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:49:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:49:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+replaying '' '' 2024-09-18T00:49:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:49:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:49:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:49:52.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:49:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:49:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:49:52.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:49:52.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 2cd3b2a7-cd9c-456d-b051-1ce989af2010 2024-09-18T00:49:52.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:49:52.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70193152.0,"last_snapshot_bytes":70193152,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620467,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"} 2024-09-18T00:49:52.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:49:52.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:39 2024-09-18T00:49:52.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:49:52.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:49:52.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:49:52.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:49:52.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:49' 2024-09-18T00:49:52.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:49:52.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:49:52.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:49:52.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70193152.0,"last_snapshot_bytes":70193152,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620467,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"} 2024-09-18T00:49:52.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:49:52.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:49:52.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:49:52.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:49:52.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:49:52.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:49:52.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:49:52.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:49:52.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:49:52.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+replaying '2024-09-18 00:49:39' 2024-09-18T00:49:52.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:52.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:52.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:49:52.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:52.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:49:39' 2024-09-18T00:49:52.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:49:52.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:49:52.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:49:52.581 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:52.576+0000 7f849c00d640 -1 reset not still connected to 0x5629c5fb4410 2024-09-18T00:49:52.581 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:52.576+0000 7f849c00d640 -1 reset not still connected to 0x5629c5fb45b0 2024-09-18T00:49:52.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK842544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:49image-primary32cd3b2a7-cd9c-456d-b051-1ce989af2010up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70193152.0,"last_snapshot_bytes":70193152,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620467,"remote_snapshot_timestamp":1726620467,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:49image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70037504.0,"last_snapshot_bytes":70037504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:49image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:49image-primary61595563f-88e0-4d3f-be52-370c261044d6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70307840.0,"last_snapshot_bytes":70307840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:49image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620462,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:49image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69959680.0,"last_snapshot_bytes":69959680,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:49image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70164480.0,"last_snapshot_bytes":70164480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:49' 2024-09-18T00:49:52.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:49:52.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:49:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:49:52.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:49:52.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:49:52.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:49:52.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:49:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:49:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:52.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary3 2024-09-18T00:49:52.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:49:52.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:52.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:49:52.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary3 2024-09-18T00:49:52.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary3 2024-09-18T00:49:52.925 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:52.920+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:49:52.938 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:52.932+0000 7f849c00d640 -1 reset not still connected to 0x5629ae79fad0 2024-09-18T00:49:52.938 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:52.936+0000 7f849c00d640 -1 reset not still connected to 0x5629bfe46680 2024-09-18T00:49:52.938 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:52.936+0000 7f849c00d640 -1 reset not still connected to 0x5629c3bd2f70 2024-09-18T00:49:52.938 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:52.936+0000 7f849c00d640 -1 reset not still connected to 0x5629c3bd3ba0 2024-09-18T00:49:52.938 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:52.936+0000 7f849c00d640 -1 reset not still connected to 0x5629e3768c30 2024-09-18T00:49:52.938 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:52.936+0000 7f849c00d640 -1 reset not still connected to 0x5629e3768f70 2024-09-18T00:49:52.938 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:52.936+0000 7f849c00d640 -1 reset not still connected to 0x5629f9af75f0 2024-09-18T00:49:53.101 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:53.092+0000 7f849c00d640 -1 reset not still connected to 0x5629e3768f70 2024-09-18T00:49:53.101 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:53.092+0000 7f849c00d640 -1 reset not still connected to 0x5629f9af75f0 2024-09-18T00:49:53.740 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:49:53.736+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4000 2024-09-18T00:49:53.740 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:49:53.736+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4750 2024-09-18T00:49:53.741 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:49:53.736+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6734b1ee0 2024-09-18T00:49:53.741 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:49:53.736+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735afee0 2024-09-18T00:49:53.741 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:49:53.736+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6a97f21a0 2024-09-18T00:49:54.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:49:54.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary3 2024-09-18T00:49:54.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:54.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:54.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:49:54.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:49:54.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:49:54.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary3 2024-09-18T00:49:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary3 2024-09-18T00:49:54.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:49:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:49:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary4 2024-09-18T00:49:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:49:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:49:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+replaying 2024-09-18T00:49:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:49:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:49:54.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:49:54.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:49:54.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:49:55.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+replaying '' '' 2024-09-18T00:49:55.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:55.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:55.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:49:55.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:55.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:49:55.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:49:55.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:49:55.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:49:55.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:49:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:49:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4ddc12ed-652e-41cb-a869-7273142e4cee 2024-09-18T00:49:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:49:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70037504.0,"last_snapshot_bytes":70037504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"} 2024-09-18T00:49:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:49:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:39 2024-09-18T00:49:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:49:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:49:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:49:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:49:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:54' 2024-09-18T00:49:55.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:49:55.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:49:55.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:49:55.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70037504.0,"last_snapshot_bytes":70037504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"} 2024-09-18T00:49:55.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:49:55.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:49:55.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:49:55.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:49:55.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:49:55.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:49:55.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:49:55.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:49:55.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:49:55.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+replaying '2024-09-18 00:49:39' 2024-09-18T00:49:55.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:49:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:49:39' 2024-09-18T00:49:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:49:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:49:55.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:49:55.539 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:49:55.536+0000 7f849c00d640 -1 reset not still connected to 0x5629cf4be0d0 2024-09-18T00:49:55.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54image-primary44ddc12ed-652e-41cb-a869-7273142e4ceeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70037504.0,"last_snapshot_bytes":70037504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54image-primary61595563f-88e0-4d3f-be52-370c261044d6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70307840.0,"last_snapshot_bytes":70307840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620462,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69959680.0,"last_snapshot_bytes":69959680,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70164480.0,"last_snapshot_bytes":70164480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54' 2024-09-18T00:49:55.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:49:55.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:49:55.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:49:55.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:49:55.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:49:55.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:49:55.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:49:55.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:49:55.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:55.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:55.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary4 2024-09-18T00:49:55.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:49:55.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:55.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:49:55.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary4 2024-09-18T00:49:55.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary4 2024-09-18T00:49:57.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:49:57.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary4 2024-09-18T00:49:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:49:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:49:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:49:57.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary4 2024-09-18T00:49:57.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:57.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary4 2024-09-18T00:49:57.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:49:57.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:49:57.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary5 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+replaying 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:49:57.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:49:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+replaying '' '' 2024-09-18T00:49:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:49:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:49:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:49:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:49:58.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:49:58.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:49:58.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:49:58.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4dabe928-cfdd-47d0-97f0-a3be00b4b2a0 2024-09-18T00:49:58.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:49:58.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"} 2024-09-18T00:49:58.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:49:58.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:39 2024-09-18T00:49:58.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:49:58.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:49:58.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:49:58.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:49:58.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:54' 2024-09-18T00:49:58.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:49:58.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:49:58.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:49:58.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"} 2024-09-18T00:49:58.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:49:58.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:49:58.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:49:58.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:49:58.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:49:58.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:49:58.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:49:58.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:49:58.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:49:58.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+replaying '2024-09-18 00:49:39' 2024-09-18T00:49:58.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:49:58.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:58.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:49:58.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:49:58.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:49:39' 2024-09-18T00:49:58.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:49:58.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:49:58.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:49:59.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK642544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54image-primary54dabe928-cfdd-47d0-97f0-a3be00b4b2a0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54image-primary61595563f-88e0-4d3f-be52-370c261044d6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70307840.0,"last_snapshot_bytes":70307840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620462,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69959680.0,"last_snapshot_bytes":69959680,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70164480.0,"last_snapshot_bytes":70164480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:54' 2024-09-18T00:49:59.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:49:59.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:49:59.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:49:59.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:49:59.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:49:59.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:49:59.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:49:59.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:49:59.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:59.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:49:59.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary5 2024-09-18T00:49:59.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:49:59.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:49:59.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:49:59.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary5 2024-09-18T00:49:59.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary5 2024-09-18T00:50:00.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:50:00.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary5 2024-09-18T00:50:00.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:00.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:00.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:50:00.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:50:00.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:50:00.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary5 2024-09-18T00:50:00.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:00.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary5 2024-09-18T00:50:00.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:00.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:50:00.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary6 2024-09-18T00:50:00.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:00.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:00.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:00.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:50:00.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+replaying 2024-09-18T00:50:00.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:00.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:00.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:00.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:00.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:50:00.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:50:00.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:50:00.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:50:01.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+replaying '' '' 2024-09-18T00:50:01.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:01.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:50:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:50:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:50:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:50:01.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:50:01.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:50:01.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 1595563f-88e0-4d3f-be52-370c261044d6 2024-09-18T00:50:01.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:50:01.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70307840.0,"last_snapshot_bytes":70307840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"} 2024-09-18T00:50:01.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:50:01.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:39 2024-09-18T00:50:01.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:50:01.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:50:01.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:50:01.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:50:01.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:59' 2024-09-18T00:50:01.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:50:01.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:50:01.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:50:01.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70307840.0,"last_snapshot_bytes":70307840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"} 2024-09-18T00:50:01.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:50:01.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:50:01.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:50:01.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:50:01.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:50:01.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:50:01.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:50:01.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:50:01.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:50:01.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+replaying '2024-09-18 00:49:39' 2024-09-18T00:50:01.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:01.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:01.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:01.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:01.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:49:39' 2024-09-18T00:50:01.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:50:01.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:50:01.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:59image-primary61595563f-88e0-4d3f-be52-370c261044d6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70307840.0,"last_snapshot_bytes":70307840,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:59image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620462,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:59image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69959680.0,"last_snapshot_bytes":69959680,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:59image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70164480.0,"last_snapshot_bytes":70164480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:49:59' 2024-09-18T00:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:50:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:50:02.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:50:02.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:50:02.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:50:02.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:50:02.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:50:02.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:50:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary6 2024-09-18T00:50:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:50:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary6 2024-09-18T00:50:02.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary6 2024-09-18T00:50:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:50:03.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary6 2024-09-18T00:50:03.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:03.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:03.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:03.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:50:03.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:50:03.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary6 2024-09-18T00:50:04.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:04.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary6 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary7 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+replaying 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:50:04.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:50:04.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:50:04.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:50:05.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+replaying '' '' 2024-09-18T00:50:05.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:05.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:05.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:05.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:05.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:50:05.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:50:05.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:50:05.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:50:05.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:50:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:50:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: ab0d841f-81f9-4384-b3bd-341cb0d2c745 2024-09-18T00:50:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:50:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620462,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"} 2024-09-18T00:50:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:50:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:39 2024-09-18T00:50:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:50:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:50:05.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:50:05.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:50:05.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:50:04' 2024-09-18T00:50:05.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:50:05.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:50:05.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:50:05.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620462,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"} 2024-09-18T00:50:05.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:50:05.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:50:05.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:50:05.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:50:05.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:50:05.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:50:05.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:50:05.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:50:05.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:50:05.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+replaying '2024-09-18 00:49:39' 2024-09-18T00:50:05.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:49:39' 2024-09-18T00:50:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:50:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:50:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:50:05.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:50:04image-primary7ab0d841f-81f9-4384-b3bd-341cb0d2c745up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70184960.0,"last_snapshot_bytes":70184960,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620462,"remote_snapshot_timestamp":1726620462,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:50:04image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69959680.0,"last_snapshot_bytes":69959680,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:50:04image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70164480.0,"last_snapshot_bytes":70164480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:50:04' 2024-09-18T00:50:05.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:50:05.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:50:05.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:50:05.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:50:05.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:50:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:50:05.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:50:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:50:05.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:05.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:05.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary7 2024-09-18T00:50:05.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:50:05.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:05.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:05.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary7 2024-09-18T00:50:05.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary7 2024-09-18T00:50:07.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:50:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary7 2024-09-18T00:50:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:50:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:50:07.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary7 2024-09-18T00:50:07.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:07.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary7 2024-09-18T00:50:07.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:07.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:50:07.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary8 2024-09-18T00:50:07.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:50:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+replaying 2024-09-18T00:50:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:50:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:50:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:50:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:50:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+replaying '' '' 2024-09-18T00:50:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:50:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:50:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:50:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:50:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:50:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:50:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8f3bd77d-71f5-46c9-8ccc-36529855a3b4 2024-09-18T00:50:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:50:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69959680.0,"last_snapshot_bytes":69959680,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"} 2024-09-18T00:50:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:50:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:49:39 2024-09-18T00:50:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:50:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:50:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:50:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:50:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:50:04' 2024-09-18T00:50:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:50:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:50:08.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:50:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69959680.0,"last_snapshot_bytes":69959680,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"} 2024-09-18T00:50:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:50:08.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:50:08.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:50:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:50:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:50:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:50:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:50:08.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:50:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:50:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+replaying '2024-09-18 00:49:39' 2024-09-18T00:50:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:49:39' 2024-09-18T00:50:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:50:08.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:50:08.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:50:09.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK342544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:50:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:50:04image-primary88f3bd77d-71f5-46c9-8ccc-36529855a3b4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":69959680.0,"last_snapshot_bytes":69959680,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620475,"remote_snapshot_timestamp":1726620475,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:50:04image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70164480.0,"last_snapshot_bytes":70164480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:49:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:50:04' 2024-09-18T00:50:09.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:50:09.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:50:09.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:49:39' 2024-09-18T00:50:09.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:50:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:50:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:50:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:50:09.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:50:09.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:09.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:09.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary8 2024-09-18T00:50:09.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:50:09.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:09.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:09.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary8 2024-09-18T00:50:09.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary8 2024-09-18T00:50:10.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:50:10.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary8 2024-09-18T00:50:10.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:10.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:10.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:10.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:50:10.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:50:10.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary8 2024-09-18T00:50:10.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:10.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary8 2024-09-18T00:50:10.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:10.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:50:10.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary9 2024-09-18T00:50:10.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:50:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+replaying 2024-09-18T00:50:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:50:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:50:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:50:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:50:11.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+replaying '' '' 2024-09-18T00:50:11.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:11.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:11.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:11.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:11.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:50:11.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:50:11.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:50:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:50:11.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:50:11.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:50:11.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 47cdb799-ea3d-491b-a019-2a4da9cb8764 2024-09-18T00:50:11.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:50:11.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70164480.0,"last_snapshot_bytes":70164480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"} 2024-09-18T00:50:11.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:50:11.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:50:09 2024-09-18T00:50:11.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:50:11.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:50:11.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:50:11.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:50:11.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:50:09' 2024-09-18T00:50:11.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:50:11.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:50:11.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:50:11.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70164480.0,"last_snapshot_bytes":70164480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"} 2024-09-18T00:50:11.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:50:11.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:50:11.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:50:11.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:50:11.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:50:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:50:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:50:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:50:11.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:50:09' 2024-09-18T00:50:11.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+replaying '2024-09-18 00:50:09' 2024-09-18T00:50:11.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:11.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:11.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:11.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:11.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:50:09' 2024-09-18T00:50:11.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:50:11.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:50:11.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:50:12.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:50:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:50:09image-primary947cdb799-ea3d-491b-a019-2a4da9cb8764up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70164480.0,"last_snapshot_bytes":70164480,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:50:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:50:09' 2024-09-18T00:50:12.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:50:12.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:50:12.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:50:09' 2024-09-18T00:50:12.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:50:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:50:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:50:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:50:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:50:12.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:12.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:12.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary9 2024-09-18T00:50:12.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:50:12.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary9 2024-09-18T00:50:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary9 2024-09-18T00:50:13.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:50:13.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary9 2024-09-18T00:50:13.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:13.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:13.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:13.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:50:13.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:50:13.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary9 2024-09-18T00:50:13.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:13.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary9 2024-09-18T00:50:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:50:13.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary10 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+replaying 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:50:13.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:50:14.125 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:50:14.116+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:50:14.125 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:50:14.116+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:50:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+replaying '' '' 2024-09-18T00:50:14.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:14.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:14.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:14.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:14.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:50:14.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:50:14.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:50:14.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:50:14.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:50:15.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:50:15.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9b75893a-9ef9-468b-ad0c-88ee663b581a 2024-09-18T00:50:15.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:50:15.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"} 2024-09-18T00:50:15.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:50:15.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:50:09 2024-09-18T00:50:15.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:50:15.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:50:15.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:50:15.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:50:15.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:50:14' 2024-09-18T00:50:15.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:50:15.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:50:15.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:50:15.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"} 2024-09-18T00:50:15.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:50:15.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:50:15.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:50:15.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:50:15.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:50:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:50:15.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:50:15.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:50:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:50:09' 2024-09-18T00:50:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+replaying '2024-09-18 00:50:09' 2024-09-18T00:50:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:50:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:50:09' 2024-09-18T00:50:15.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:50:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:50:15.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:50:15.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary109b75893a-9ef9-468b-ad0c-88ee663b581aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70062080.0,"last_snapshot_bytes":70062080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620472,"remote_snapshot_timestamp":1726620472,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:50:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:50:14' 2024-09-18T00:50:15.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:50:15.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:50:15.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:50:09' 2024-09-18T00:50:15.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:50:15.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:50:15.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:50:15.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:50:15.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:50:15.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:15.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:15.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary10 2024-09-18T00:50:15.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:50:15.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:15.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:15.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary10 2024-09-18T00:50:15.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary10 2024-09-18T00:50:16.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:50:16.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary10 2024-09-18T00:50:16.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:16.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:16.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:16.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:50:16.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:50:16.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary10 2024-09-18T00:50:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:16.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary10 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..10} 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS=() 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS=() 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS=() 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS=() 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RET=0 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary1 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary1 snapshot 10G 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:16.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary1 10G 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:50:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:50:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:50:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary1 2024-09-18T00:50:16.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:50:16.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:16.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:50:16.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:16.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary1 snapshot 2024-09-18T00:50:16.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:16.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:16.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary1 snapshot 2024-09-18T00:50:17.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:50:17.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary1 2024-09-18T00:50:17.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary1': 2024-09-18T00:50:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:50:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:50:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:50:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 30512d5ae29d 2024-09-18T00:50:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.30512d5ae29d 2024-09-18T00:50:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:50:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:50:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:50:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:50:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:50:16 2024 2024-09-18T00:50:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:50:16 2024 2024-09-18T00:50:17.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:50:16 2024 2024-09-18T00:50:17.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:50:17.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:50:17.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 9a782482-1483-42e7-a43c-95bec4be0376 2024-09-18T00:50:17.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:50:17.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:50:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd0 2024-09-18T00:50:17.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:50:17.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd0 2024-09-18T00:50:17.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:50:18.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:50:18.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:50:18.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 7c9e766f-342c-4318-96af-7046026e9066 2024-09-18T00:50:18.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:50:18.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:50:18.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:18.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:50:18.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:50:18.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:50:18.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:50:18.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:18.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary1 2024-09-18T00:50:18.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd0 test-primary1 2024-09-18T00:50:18.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:50:18.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary1 2024-09-18T00:50:18.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:50:18.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary1 2024-09-18T00:50:18.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:18.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:18.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary1 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary2 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary2 snapshot 10G 2024-09-18T00:50:18.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary1 2024-09-18T00:50:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary1 2024-09-18T00:50:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:50:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary2 10G 2024-09-18T00:50:18.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:18.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:18.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:18.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:18.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:18.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:18.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:50:18.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:50:18.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:50:18.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:18.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary2 2024-09-18T00:50:18.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:50:18.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:50:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary2 snapshot 2024-09-18T00:50:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary2 snapshot 2024-09-18T00:50:19.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2316 2024-09-18T00:50:19.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:50:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:50:19.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary1/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary1' 2024-09-18T00:50:19.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary2 2024-09-18T00:50:19.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary2': 2024-09-18T00:50:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:50:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:50:19.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 3061e6b19218 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.3061e6b19218 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:50:18 2024 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:50:18 2024 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:50:18 2024 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: a96adece-8cff-4940-898a-d2a762b46aea 2024-09-18T00:50:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:50:19.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:50:19.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:50:20.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd1 2024-09-18T00:50:20.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:50:20.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd1 2024-09-18T00:50:20.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:50:20.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:50:20.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:50:20.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 07763fc3-3d97-4ac1-8ada-66fafb0c39cf 2024-09-18T00:50:20.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:50:20.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:50:20.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:20.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:50:20.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:50:20.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:50:20.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:50:20.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:20.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary2 2024-09-18T00:50:20.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd1 test-primary2 2024-09-18T00:50:20.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:50:20.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary2 2024-09-18T00:50:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:50:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary2 2024-09-18T00:50:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:50:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary2 2024-09-18T00:50:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:20.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:50:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:50:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary3 2024-09-18T00:50:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary3 snapshot 10G 2024-09-18T00:50:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:50:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary2 2024-09-18T00:50:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary2 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary3 10G 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:20.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:20.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:50:20.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:20.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:50:20.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:50:20.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:50:20.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:20.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary3 2024-09-18T00:50:20.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:50:20.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:20.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:50:20.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:20.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary3 snapshot 2024-09-18T00:50:20.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:20.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:20.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:50:20.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:20.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary3 snapshot 2024-09-18T00:50:20.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:50:20.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary2/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary2' 2024-09-18T00:50:21.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2318 2024-09-18T00:50:21.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:50:21.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary3 2024-09-18T00:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary3': 2024-09-18T00:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 30716f96b7f0 2024-09-18T00:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.30716f96b7f0 2024-09-18T00:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:50:20 2024 2024-09-18T00:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:50:20 2024 2024-09-18T00:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:50:20 2024 2024-09-18T00:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 238293d6-165c-443a-acb3-b579f41e40a2 2024-09-18T00:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:50:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:50:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:50:21.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd2 2024-09-18T00:50:21.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:50:21.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd2 2024-09-18T00:50:21.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:50:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:50:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:50:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 7e6efce9-c18b-4d0a-841d-93842a937bdb 2024-09-18T00:50:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:50:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:50:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:50:21.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:50:22.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:50:22.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:50:22.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:22.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary3 2024-09-18T00:50:22.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd2 test-primary3 2024-09-18T00:50:22.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:50:22.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary3 2024-09-18T00:50:22.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:50:22.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary3 2024-09-18T00:50:22.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:22.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary3 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary4 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary3 2024-09-18T00:50:22.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary4 snapshot 10G 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary3 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary4 10G 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:50:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:50:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:50:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:50:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:22.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary4 2024-09-18T00:50:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:50:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:50:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary4 snapshot 2024-09-18T00:50:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:50:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary4 snapshot 2024-09-18T00:50:22.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:50:22.667 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary3/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary3' 2024-09-18T00:50:22.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:22.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:50:22.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:22.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:22.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:22.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:50:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2320 2024-09-18T00:50:22.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:22.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:50:22.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary4 2024-09-18T00:50:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary4': 2024-09-18T00:50:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:50:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:50:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:50:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 307f73e2319b 2024-09-18T00:50:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.307f73e2319b 2024-09-18T00:50:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:50:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:50:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:50:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:50:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:50:22 2024 2024-09-18T00:50:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:50:22 2024 2024-09-18T00:50:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:50:22 2024 2024-09-18T00:50:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:50:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:50:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: cd38d20d-4581-4726-a3c5-e2f4f19b40f5 2024-09-18T00:50:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:50:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:50:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:50:23.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd3 2024-09-18T00:50:23.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:50:23.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd3 2024-09-18T00:50:23.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:50:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:50:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:50:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: caed63b4-7371-489a-a0ed-4587a877f604 2024-09-18T00:50:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:50:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:50:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:50:23.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:50:23.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:50:23.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:50:23.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary4 2024-09-18T00:50:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd3 test-primary4 2024-09-18T00:50:23.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:50:23.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary4 2024-09-18T00:50:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:50:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary4 2024-09-18T00:50:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:50:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:50:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:50:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:50:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:23.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary5 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary4 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary5 snapshot 10G 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:50:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary4 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary5 10G 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary4 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:23.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary5 2024-09-18T00:50:23.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:50:23.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:23.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:50:23.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:23.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary5 snapshot 2024-09-18T00:50:23.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:23.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:23.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:50:23.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:23.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary5 snapshot 2024-09-18T00:50:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:50:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary4/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary4' 2024-09-18T00:50:24.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:24.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:50:24.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:24.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:24.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:24.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:50:24.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2322 2024-09-18T00:50:24.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:24.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2323 2024-09-18T00:50:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:50:24.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary5 2024-09-18T00:50:25.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary5': 2024-09-18T00:50:25.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:50:25.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:50:25.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:50:25.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 309163f6b35e 2024-09-18T00:50:25.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.309163f6b35e 2024-09-18T00:50:25.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:50:25.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:50:25.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:50:25.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:50:25.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:50:23 2024 2024-09-18T00:50:25.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:50:23 2024 2024-09-18T00:50:25.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:50:23 2024 2024-09-18T00:50:25.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:50:25.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:50:25.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 8b737e05-c466-4ab1-8f79-d9dc4c934fea 2024-09-18T00:50:25.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:50:25.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:50:25.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:50:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd4 2024-09-18T00:50:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:50:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd4 2024-09-18T00:50:25.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:50:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:50:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:50:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 135b7c4f-c9eb-4dbf-9b80-34d884570ac5 2024-09-18T00:50:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:50:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:50:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:50:25.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:50:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:50:25.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:50:25.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:25.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary5 2024-09-18T00:50:25.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd4 test-primary5 2024-09-18T00:50:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2325 2024-09-18T00:50:25.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:25.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:25.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:50:25.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:25.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:25.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:50:25.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:50:25.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:50:25.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary5 2024-09-18T00:50:25.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:50:25.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary5 2024-09-18T00:50:25.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:50:25.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:25.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:25.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:50:25.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary5 2024-09-18T00:50:25.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary6 2024-09-18T00:50:25.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary6 snapshot 10G 2024-09-18T00:50:25.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:25.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary5 2024-09-18T00:50:25.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary6 10G 2024-09-18T00:50:25.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary5 2024-09-18T00:50:25.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:25.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:25.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:25.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:25.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:50:25.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:25.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:25.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:25.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:50:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:50:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:50:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:50:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:25.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary6 2024-09-18T00:50:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:50:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:50:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary6 snapshot 2024-09-18T00:50:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:26.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary6 snapshot 2024-09-18T00:50:26.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:50:26.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary5/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary5' 2024-09-18T00:50:26.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:50:26.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2327 2024-09-18T00:50:26.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:26.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary6 2024-09-18T00:50:27.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary6': 2024-09-18T00:50:27.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:50:27.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:50:27.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:50:27.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 30a3f7674d05 2024-09-18T00:50:27.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.30a3f7674d05 2024-09-18T00:50:27.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:50:27.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:50:27.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:50:27.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:50:27.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:50:26 2024 2024-09-18T00:50:27.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:50:26 2024 2024-09-18T00:50:27.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:50:26 2024 2024-09-18T00:50:27.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:50:27.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:50:27.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 77b516d0-a26f-44c3-b75d-8945de8e9052 2024-09-18T00:50:27.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:50:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:50:27.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:50:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd5 2024-09-18T00:50:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:50:27.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd5 2024-09-18T00:50:27.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:50:27.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:50:27.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:50:27.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: bce56850-aaee-41b3-8a57-be18a30e1708 2024-09-18T00:50:27.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:50:27.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:50:27.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:27.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:50:27.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:50:27.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:50:27.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:50:27.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:27.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary6 2024-09-18T00:50:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd5 test-primary6 2024-09-18T00:50:27.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:50:27.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary6 2024-09-18T00:50:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:50:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:50:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary6 2024-09-18T00:50:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:50:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary7 2024-09-18T00:50:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary7 snapshot 10G 2024-09-18T00:50:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary6 2024-09-18T00:50:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary6 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary7 10G 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:27.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:27.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:27.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:27.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:50:27.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:50:27.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:50:27.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:27.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary7 2024-09-18T00:50:27.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary6 2024-09-18T00:50:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:50:27.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:27.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:27.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:27.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:50:27.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:27.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:50:27.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:27.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:27.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:27.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:50:27.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2326 2024-09-18T00:50:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:50:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary6/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary6' 2024-09-18T00:50:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:50:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:50:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary7 snapshot 2024-09-18T00:50:27.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary7 snapshot 2024-09-18T00:50:27.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:50:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:50:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:50:28.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:28.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:50:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:50:28.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2329 2024-09-18T00:50:28.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:29.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:50:29.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary7 2024-09-18T00:50:29.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary7': 2024-09-18T00:50:29.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:50:29.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:50:29.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:50:29.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 30b3fe1702f4 2024-09-18T00:50:29.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.30b3fe1702f4 2024-09-18T00:50:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:50:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:50:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:50:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:50:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:50:27 2024 2024-09-18T00:50:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:50:27 2024 2024-09-18T00:50:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:50:27 2024 2024-09-18T00:50:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:50:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:50:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 682d0054-0ced-419a-bc1c-f9e86132b9fc 2024-09-18T00:50:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:50:29.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:50:29.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:50:29.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd6 2024-09-18T00:50:29.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:50:29.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd6 2024-09-18T00:50:29.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:50:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:50:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:50:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 301bb6eb-2593-47b5-9881-9d1c2e8b2f62 2024-09-18T00:50:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:50:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:50:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:50:29.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:50:29.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): Snapshot ID: 2330 2024-09-18T00:50:29.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:29.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:29.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:50:29.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:29.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:29.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:50:29.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:50:29.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:50:30.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 Snapshot ID: 2332 2024-09-18T00:50:30.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:30.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:50:30.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:30.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary7 2024-09-18T00:50:30.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd6 test-primary7 2024-09-18T00:50:30.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:50:30.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary7 2024-09-18T00:50:30.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:50:30.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary7 2024-09-18T00:50:30.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:30.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:30.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary7 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary8 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary8 snapshot 10G 2024-09-18T00:50:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary7 2024-09-18T00:50:30.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary7 2024-09-18T00:50:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:30.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:50:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary8 10G 2024-09-18T00:50:30.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:50:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:50:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:50:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary8 2024-09-18T00:50:30.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:50:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:50:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary8 snapshot 2024-09-18T00:50:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary8 snapshot 2024-09-18T00:50:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:50:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary7/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary7' 2024-09-18T00:50:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:50:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:50:30.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:50:30.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2333 2024-09-18T00:50:30.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:31.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2335 2024-09-18T00:50:31.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:50:31.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:31.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary8 2024-09-18T00:50:31.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary8': 2024-09-18T00:50:31.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:50:31.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:50:31.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 30cdefdd14b0 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.30cdefdd14b0 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:50:30 2024 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:50:30 2024 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:50:30 2024 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: c183a554-660d-4696-b42f-a3c4a75ef26e 2024-09-18T00:50:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:50:31.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:50:31.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:50:31.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd7 2024-09-18T00:50:31.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:50:31.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd7 2024-09-18T00:50:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:50:31.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:50:31.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:50:31.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 0510d441-e024-4355-b489-4f9cadfb1395 2024-09-18T00:50:31.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:50:31.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:50:31.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:31.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:50:31.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:50:31.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:50:31.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:50:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:50:31.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:50:31.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:31.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary8 2024-09-18T00:50:31.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd7 test-primary8 2024-09-18T00:50:31.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2334 2024-09-18T00:50:32.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:32.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:50:32.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary8 2024-09-18T00:50:32.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:50:32.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary8 2024-09-18T00:50:32.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:50:32.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:32.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:32.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:32.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:32.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary8 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary9 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary9 snapshot 10G 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary8 2024-09-18T00:50:32.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary8 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary9 10G 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:32.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:50:32.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:50:32.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:50:32.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:32.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary9 2024-09-18T00:50:32.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:50:32.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:32.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:50:32.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:32.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary9 snapshot 2024-09-18T00:50:32.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:32.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:32.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:32.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:32.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary9 snapshot 2024-09-18T00:50:32.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:50:32.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary8/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary8' 2024-09-18T00:50:32.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2337 2024-09-18T00:50:32.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:32.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2339 2024-09-18T00:50:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:50:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:32.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:32.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:50:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:32.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:50:33.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary9 2024-09-18T00:50:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:50:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:50:33.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:50:33.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary9': 2024-09-18T00:50:33.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:50:33.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:50:33.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:50:33.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 30e0f41d1894 2024-09-18T00:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.30e0f41d1894 2024-09-18T00:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:50:32 2024 2024-09-18T00:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:50:32 2024 2024-09-18T00:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:50:32 2024 2024-09-18T00:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 93ec4895-421e-451c-941b-0c6809cf872e 2024-09-18T00:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:50:33.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:50:33.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:50:33.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd8 2024-09-18T00:50:33.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:50:33.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd8 2024-09-18T00:50:33.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:50:33.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:50:33.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:50:33.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: c0ab2b8d-52d8-40a4-91aa-463b02b017eb 2024-09-18T00:50:33.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:50:33.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:50:33.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:33.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:50:33.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:50:33.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:50:33.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:50:33.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:33.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary9 2024-09-18T00:50:33.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd8 test-primary9 2024-09-18T00:50:33.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:50:33.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary9 2024-09-18T00:50:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:50:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary9 2024-09-18T00:50:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary9 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:50:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary10 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary10 snapshot 10G 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary9 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary9 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:33.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary10 10G 2024-09-18T00:50:33.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:33.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:33.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:33.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:33.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:33.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:33.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:50:33.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:50:33.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:50:33.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:50:33.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary10 2024-09-18T00:50:33.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:50:33.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:33.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:50:33.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:50:33.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary10 snapshot 2024-09-18T00:50:33.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:33.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:33.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:33.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:50:33.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary10 snapshot 2024-09-18T00:50:33.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:50:33.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary9/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary9' 2024-09-18T00:50:33.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2338 2024-09-18T00:50:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:50:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:33.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:50:33.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:33.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2343 2024-09-18T00:50:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:34.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:34.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:50:34.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:34.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:34.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:34.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:50:34.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2342 2024-09-18T00:50:34.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:34.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:50:35.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:35.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:50:35.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:35.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:35.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:50:35.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:50:35.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary10 2024-09-18T00:50:35.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2341 2024-09-18T00:50:35.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:35.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary10': 2024-09-18T00:50:35.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 30f568ba1f32 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.30f568ba1f32 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:50:33 2024 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:50:33 2024 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:50:33 2024 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:50:35.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:50:35.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 6f07d2c4-2b18-4dec-be82-bb3768331b71 2024-09-18T00:50:35.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:50:35.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:50:35.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:50:35.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd9 2024-09-18T00:50:35.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:50:35.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd9 2024-09-18T00:50:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:50:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:50:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:50:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 3f3dda30-1d99-4d09-8f40-4ceea82b7c7c 2024-09-18T00:50:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:50:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:50:35.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:35.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:50:35.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:50:35.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:50:35.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:35.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:50:35.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:35.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:35.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:50:35.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:50:35.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:35.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:50:35.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:35.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:35.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:35.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:50:35.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 Snapshot ID: 2346 2024-09-18T00:50:35.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:35.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2345 2024-09-18T00:50:36.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:50:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:50:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary10 2024-09-18T00:50:36.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd9 test-primary10 2024-09-18T00:50:36.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:50:36.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary10 2024-09-18T00:50:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:50:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary10 2024-09-18T00:50:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:50:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:36.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:50:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary10 2024-09-18T00:50:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:50:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 108581 2024-09-18T00:50:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:50:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary10 2024-09-18T00:50:36.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary10 2024-09-18T00:50:36.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:50:36.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary10/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary10' 2024-09-18T00:50:36.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2347 2024-09-18T00:50:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:50:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:36.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:50:36.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:50:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:36.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:50:36.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2350 2024-09-18T00:50:37.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:37.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2348 2024-09-18T00:50:37.853 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:37.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2349 2024-09-18T00:50:37.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:50:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:50:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:50:38.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2352 2024-09-18T00:50:38.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:38.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2351 2024-09-18T00:50:38.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:38.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:50:38.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:38.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:38.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:38.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:50:38.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:39.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:39.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:50:39.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:39.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:39.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:39.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:50:39.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:39.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:50:39.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:39.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:39.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:50:39.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:50:39.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2354 2024-09-18T00:50:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2353 2024-09-18T00:50:40.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:40.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:50:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:40.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:50:40.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:40.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:50:40.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:40.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:40.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:50:40.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:50:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:50:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:50:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2355 2024-09-18T00:50:40.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2356 2024-09-18T00:50:40.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:40.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:41.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:41.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:50:41.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:41.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:41.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:41.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:50:41.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2358 2024-09-18T00:50:41.985 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:50:41.983+0000 7f52d7c6c640 -1 reset not still connected to 0x55784a12a9c0 2024-09-18T00:50:41.985 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:50:41.983+0000 7f52d7c6c640 -1 reset not still connected to 0x557872bb4dd0 2024-09-18T00:50:41.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:41.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:50:41.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:41.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:41.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:41.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:50:41.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:42.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2357 2024-09-18T00:50:42.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:42.038 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:50:42.035+0000 7f849c00d640 -1 reset not still connected to 0x5629c27f7520 2024-09-18T00:50:42.039 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:50:42.035+0000 7f849c00d640 -1 reset not still connected to 0x5629c3616820 2024-09-18T00:50:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2360 2024-09-18T00:50:42.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2359 2024-09-18T00:50:42.852 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:42.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:43.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:43.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:50:43.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:43.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:43.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:43.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:50:43.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:43.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:50:43.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:43.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:43.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:50:43.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:50:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2361 2024-09-18T00:50:43.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2362 2024-09-18T00:50:43.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:43.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:43.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:50:43.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:43.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:43.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:43.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:50:43.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:43.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:43.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:50:43.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:43.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:43.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:43.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:50:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2363 2024-09-18T00:50:44.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2364 2024-09-18T00:50:44.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:45.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:45.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:50:45.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:45.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:45.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:45.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:50:45.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:45.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:45.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:50:45.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:45.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:45.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:50:45.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:45.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:50:45.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:45.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:50:45.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:45.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:45.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:50:45.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:50:45.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2366 2024-09-18T00:50:45.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2365 2024-09-18T00:50:45.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:46.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:50:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:50:46.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2367 2024-09-18T00:50:46.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:50:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:50:46.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:47.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2368 2024-09-18T00:50:47.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:47.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2370 2024-09-18T00:50:47.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:50:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:50:48.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:50:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:50:48.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:50:48.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:48.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:50:48.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:48.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:48.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:48.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:50:49.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:49.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:50:49.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:49.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:49.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:49.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:50:49.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2372 2024-09-18T00:50:49.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2373 2024-09-18T00:50:49.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:49.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2371 2024-09-18T00:50:49.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:49.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:49.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:49.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:50:49.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:49.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:49.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:49.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:50:50.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2375 2024-09-18T00:50:50.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:50.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:50:50.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:50.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:50.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:50:50.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:50:50.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2374 2024-09-18T00:50:50.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:50.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:50.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:50:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:50:50.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:50:51.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2376 2024-09-18T00:50:51.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2377 2024-09-18T00:50:51.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:51.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:51.759 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:50:51.758+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f89c0 2024-09-18T00:50:51.759 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:50:51.758+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T00:50:51.759 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:50:51.758+0000 7f52d7c6c640 -1 reset not still connected to 0x557844c39c70 2024-09-18T00:50:51.759 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:50:51.758+0000 7f52d7c6c640 -1 reset not still connected to 0x5578469ea0d0 2024-09-18T00:50:51.759 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:50:51.758+0000 7f52d7c6c640 -1 reset not still connected to 0x557846a424e0 2024-09-18T00:50:51.759 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:50:51.758+0000 7f52d7c6c640 -1 reset not still connected to 0x5578470b6340 2024-09-18T00:50:51.760 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:50:51.758+0000 7f52d7c6c640 -1 reset not still connected to 0x557848aa9ba0 2024-09-18T00:50:51.760 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:50:51.758+0000 7f52d7c6c640 -1 reset not still connected to 0x55784a12a9c0 2024-09-18T00:50:51.763 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:50:51.758+0000 7f52d7c6c640 -1 reset not still connected to 0x557872bb4dd0 2024-09-18T00:50:52.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2378 2024-09-18T00:50:52.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2379 2024-09-18T00:50:52.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:52.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:50:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:50:52.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:52.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:50:52.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:52.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:52.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:52.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:50:52.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:52.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:50:52.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:52.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:52.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:52.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:50:52.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2380 2024-09-18T00:50:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:53.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:53.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:50:53.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:53.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:53.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:50:53.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:50:53.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:53.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:50:53.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:53.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:53.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:53.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:50:54.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2381 2024-09-18T00:50:54.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:54.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:50:54.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:54.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:54.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:54.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:50:54.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:54.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:50:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:54.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:50:54.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2382 2024-09-18T00:50:54.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:54.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2383 2024-09-18T00:50:54.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:54.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:55.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2385 2024-09-18T00:50:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2384 2024-09-18T00:50:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:55.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:50:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:50:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:50:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:55.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:55.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:50:55.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:55.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:55.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:50:55.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:50:55.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:55.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:50:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:50:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:50:56.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2386 2024-09-18T00:50:56.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2387 2024-09-18T00:50:56.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:56.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:57.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2388 2024-09-18T00:50:57.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:57.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2390 2024-09-18T00:50:57.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:57.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:50:57.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:57.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:57.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:50:57.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:50:57.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2389 2024-09-18T00:50:57.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:50:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:50:57.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:50:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:57.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:57.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:50:57.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:57.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:50:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:50:58.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:58.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:50:58.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:58.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:58.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:50:58.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:50:58.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:58.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:50:58.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:58.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:58.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:50:58.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:50:58.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2391 2024-09-18T00:50:58.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2392 2024-09-18T00:50:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:58.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2393 2024-09-18T00:50:58.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:58.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:50:59.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:59.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:50:59.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:59.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:59.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:50:59.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:50:59.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:50:59.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:50:59.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:50:59.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:50:59.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:50:59.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2394 2024-09-18T00:51:00.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:00.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:51:00.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:00.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:00.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:51:00.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:51:00.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2395 2024-09-18T00:51:00.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:51:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:51:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:51:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:00.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:00.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:51:00.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:00.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:00.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:51:00.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:51:01.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2397 2024-09-18T00:51:01.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:01.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2396 2024-09-18T00:51:01.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:01.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:01.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:51:01.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:01.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:01.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:51:01.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:51:01.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:01.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:51:01.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:01.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:01.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:51:01.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:51:01.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:01.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:51:01.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:01.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:01.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:51:01.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:51:01.950 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:51:01.950+0000 7f849c00d640 -1 reset not still connected to 0x5629c27f7520 2024-09-18T00:51:01.954 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:51:01.954+0000 7f849c00d640 -1 reset not still connected to 0x5629c3616820 2024-09-18T00:51:02.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2399 2024-09-18T00:51:02.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2398 2024-09-18T00:51:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2400 2024-09-18T00:51:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:02.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:02.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:03.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:03.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:51:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:51:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:51:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:51:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:51:03.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:51:03.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2401 2024-09-18T00:51:03.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2402 2024-09-18T00:51:03.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2403 2024-09-18T00:51:03.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:03.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:03.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:51:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:51:04.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:04.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:51:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:51:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:51:04.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2404 2024-09-18T00:51:04.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2405 2024-09-18T00:51:04.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:04.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:51:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:51:05.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:51:05.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:05.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:51:05.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:05.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:05.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:51:05.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:51:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:51:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:51:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:51:05.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2407 2024-09-18T00:51:05.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2406 2024-09-18T00:51:05.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:05.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:06.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2410 2024-09-18T00:51:06.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2408 2024-09-18T00:51:06.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2409 2024-09-18T00:51:06.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:06.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:06.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:51:06.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:06.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:06.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:51:06.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:51:06.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:06.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:06.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:51:06.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:06.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:06.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:51:06.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:51:06.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:06.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:06.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:51:06.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:06.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:06.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:51:06.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:51:07.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:07.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:51:07.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:07.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:07.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:51:07.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:51:07.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:07.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:51:07.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:07.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:07.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:51:07.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:51:08.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:08.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:51:08.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:08.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:08.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:51:08.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:51:08.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:08.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:51:08.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:08.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:08.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:51:08.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:08.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2413 2024-09-18T00:51:08.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2411 2024-09-18T00:51:08.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2412 2024-09-18T00:51:08.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:09.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:09.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:51:09.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:09.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:09.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:51:09.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:51:09.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2415 2024-09-18T00:51:09.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:09.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:51:09.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:09.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:09.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:51:09.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:51:09.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2414 2024-09-18T00:51:09.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:09.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:51:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:51:09.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:51:10.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2416 2024-09-18T00:51:10.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2417 2024-09-18T00:51:10.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:10.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:11.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2419 2024-09-18T00:51:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2418 2024-09-18T00:51:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:11.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2420 2024-09-18T00:51:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:51:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:51:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:51:11.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:51:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:51:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:51:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:11.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:11.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:51:11.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:11.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:11.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:51:11.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:51:12.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:12.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:51:12.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:12.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:12.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:51:12.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:51:12.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:12.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:51:12.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:12.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:12.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:51:12.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:51:13.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2421 2024-09-18T00:51:13.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:13.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:51:13.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:13.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:13.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:51:13.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:51:13.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2423 2024-09-18T00:51:13.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:13.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2422 2024-09-18T00:51:13.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:13.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:51:13.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:13.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:13.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:51:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:13.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:13.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:13.993 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:51:13.994+0000 7f849c00d640 -1 reset not still connected to 0x5629c27f7520 2024-09-18T00:51:14.001 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:51:13.994+0000 7f849c00d640 -1 reset not still connected to 0x5629c3616820 2024-09-18T00:51:14.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:14.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:51:14.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:14.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:14.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:51:14.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:51:14.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2424 2024-09-18T00:51:14.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:14.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:51:14.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:14.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:14.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:51:14.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:51:14.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2425 2024-09-18T00:51:14.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:14.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:51:14.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:14.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:14.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:51:14.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:51:14.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:14.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:15.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2426 2024-09-18T00:51:15.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2427 2024-09-18T00:51:15.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:16.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:51:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:51:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:51:16.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:16.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:51:16.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:16.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:16.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:51:16.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:51:16.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2428 2024-09-18T00:51:16.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2430 2024-09-18T00:51:16.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:16.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:51:16.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:16.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:16.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:51:16.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:51:16.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2429 2024-09-18T00:51:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:17.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:17.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:17.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:17.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:51:17.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:17.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:17.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:51:17.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:51:17.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:17.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:51:17.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:17.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:17.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:51:17.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:51:18.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2431 2024-09-18T00:51:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2432 2024-09-18T00:51:18.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:18.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:51:18.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:18.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:18.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:51:18.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:51:19.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:19.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:51:19.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:19.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:19.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:51:19.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:19.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:19.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2433 2024-09-18T00:51:19.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:19.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:19.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2435 2024-09-18T00:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:51:20.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:20.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:51:20.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:20.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:20.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:51:20.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:51:20.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:20.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:51:20.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:20.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:20.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:51:20.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:51:20.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2434 2024-09-18T00:51:20.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:21.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2437 2024-09-18T00:51:21.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2436 2024-09-18T00:51:21.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:22.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:22.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:51:22.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:22.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:22.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:51:22.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:51:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:51:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:51:22.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:51:22.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2438 2024-09-18T00:51:22.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2439 2024-09-18T00:51:22.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2440 2024-09-18T00:51:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:51:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:51:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:51:22.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:22.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:22.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:23.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:23.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:51:23.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:23.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:23.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:51:23.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:51:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:51:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:51:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:51:23.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2443 2024-09-18T00:51:23.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2441 2024-09-18T00:51:23.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:23.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2442 2024-09-18T00:51:23.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:23.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:24.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:24.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:51:24.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:24.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:24.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:51:24.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:24.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:24.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:51:24.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:24.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:24.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:51:24.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:51:24.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2444 2024-09-18T00:51:25.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2445 2024-09-18T00:51:25.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:25.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:25.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:25.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:51:25.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:25.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:25.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:51:25.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:51:25.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:25.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:51:25.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:25.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:25.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:51:25.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:51:25.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:25.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:51:25.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:25.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:25.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:51:25.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:51:26.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2446 2024-09-18T00:51:26.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2447 2024-09-18T00:51:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:26.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:51:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:51:26.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:51:26.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:26.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:51:26.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:26.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:26.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:51:26.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:51:26.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:26.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:51:26.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:26.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:26.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:51:26.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:51:27.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2448 2024-09-18T00:51:27.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2450 2024-09-18T00:51:27.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2449 2024-09-18T00:51:27.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:51:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:51:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:51:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:51:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:51:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:51:28.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2452 2024-09-18T00:51:28.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2451 2024-09-18T00:51:28.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:28.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2453 2024-09-18T00:51:28.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:28.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:29.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:29.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:51:29.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:29.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:29.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:51:29.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:29.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:29.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:51:29.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:29.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:29.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:51:29.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:51:30.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2454 2024-09-18T00:51:30.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2455 2024-09-18T00:51:30.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:30.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:51:30.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:30.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:30.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:51:30.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:51:30.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:30.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:51:30.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:30.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:30.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:51:30.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:51:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:30.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:30.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:51:30.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:30.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:30.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:51:30.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:51:30.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:31.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2457 2024-09-18T00:51:31.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2456 2024-09-18T00:51:31.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:31.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:51:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:51:31.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:51:31.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:31.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:51:31.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:31.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:31.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:51:31.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:51:31.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:31.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:51:31.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:31.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:31.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:51:31.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:51:32.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2458 2024-09-18T00:51:32.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:32.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2460 2024-09-18T00:51:32.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2459 2024-09-18T00:51:32.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:32.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:51:33.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:33.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:51:33.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:33.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:33.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:51:33.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:51:33.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2461 2024-09-18T00:51:33.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:33.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2463 2024-09-18T00:51:33.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2462 2024-09-18T00:51:33.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:33.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:34.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:34.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:51:34.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:34.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:34.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:51:34.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:51:34.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:34.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:51:34.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:34.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:34.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:51:34.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:35.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:35.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:51:35.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:35.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:35.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:51:35.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:51:35.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2464 2024-09-18T00:51:35.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2465 2024-09-18T00:51:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:51:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:51:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:51:35.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:35.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:51:35.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:35.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:35.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:51:35.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:51:35.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:35.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:36.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2466 2024-09-18T00:51:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:36.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2467 2024-09-18T00:51:36.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:36.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:36.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:51:36.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:36.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:51:36.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:51:36.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:36.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:51:36.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:36.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:36.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:51:36.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:51:36.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:51:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:51:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:51:37.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2469 2024-09-18T00:51:37.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2468 2024-09-18T00:51:37.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:37.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:37.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2470 2024-09-18T00:51:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:38.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:51:38.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:38.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:38.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:51:38.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:51:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:51:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:51:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:51:38.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2471 2024-09-18T00:51:38.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2472 2024-09-18T00:51:38.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2473 2024-09-18T00:51:38.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:39.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:39.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:51:39.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:39.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:39.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:51:39.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:51:39.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:39.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:51:39.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:39.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:39.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:51:39.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:39.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2475 2024-09-18T00:51:39.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2474 2024-09-18T00:51:39.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:39.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:40.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:40.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:51:40.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:40.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:40.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:51:40.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:40.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:51:40.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:40.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:51:40.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:40.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:51:40.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:51:40.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:40.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:51:40.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:40.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:40.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:51:40.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:51:40.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2477 2024-09-18T00:51:40.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2476 2024-09-18T00:51:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:41.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2478 2024-09-18T00:51:41.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:41.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2479 2024-09-18T00:51:41.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:41.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:51:41.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:41.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:41.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:51:41.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:51:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:51:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:51:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:51:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:51:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:51:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:51:41.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:41.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2480 2024-09-18T00:51:42.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:42.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:42.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:51:42.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:42.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:42.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:51:42.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:51:42.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:42.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:51:42.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:42.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:42.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:51:42.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:51:43.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:43.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:51:43.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:43.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:43.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:51:43.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:51:43.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2481 2024-09-18T00:51:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:51:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:51:43.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2482 2024-09-18T00:51:43.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2483 2024-09-18T00:51:43.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:43.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:43.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:44.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2484 2024-09-18T00:51:44.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2485 2024-09-18T00:51:44.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:51:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:51:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:51:44.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:51:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:51:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:51:45.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:45.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:51:45.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:45.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:45.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:51:45.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:51:45.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2487 2024-09-18T00:51:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2486 2024-09-18T00:51:45.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:45.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:46.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2488 2024-09-18T00:51:46.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2490 2024-09-18T00:51:46.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:46.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2489 2024-09-18T00:51:46.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:46.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:46.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:46.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:51:46.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:46.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:46.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:51:46.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:51:46.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:46.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:51:46.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:46.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:46.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:51:46.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:51:46.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:46.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:51:46.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:46.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:46.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:51:46.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:51:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:51:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:51:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:51:47.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:47.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:51:47.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:47.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:47.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:51:47.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:51:48.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2491 2024-09-18T00:51:48.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2492 2024-09-18T00:51:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:48.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:51:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:51:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:48.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:48.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:51:48.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:48.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:48.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:51:48.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:51:48.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2493 2024-09-18T00:51:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:51:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:51:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:51:49.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2494 2024-09-18T00:51:49.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:49.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:51:49.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:49.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:49.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:51:49.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:51:49.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:49.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:51:49.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:49.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:49.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:51:49.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:51:49.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2495 2024-09-18T00:51:49.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:49.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:50.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2497 2024-09-18T00:51:50.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2496 2024-09-18T00:51:50.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:51.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:51.647 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:51:51.637+0000 7f52d7c6c640 -1 reset not still connected to 0x557848aa9ba0 2024-09-18T00:51:51.648 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:51:51.637+0000 7f52d7c6c640 -1 reset not still connected to 0x55784a12a9c0 2024-09-18T00:51:51.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2499 2024-09-18T00:51:51.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2500 2024-09-18T00:51:51.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2498 2024-09-18T00:51:51.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:51:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:51:51.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:51:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:51.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:51.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:51:51.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:51.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:51.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:51:51.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:51:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:51:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:52.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:51:52.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:51:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:52.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:51:52.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:52.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:52.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:51:52.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:51:52.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:52.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:51:52.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:52.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:52.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:51:52.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:51:53.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2501 2024-09-18T00:51:53.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:53.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2502 2024-09-18T00:51:53.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:53.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:51:53.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:53.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:51:53.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:51:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2503 2024-09-18T00:51:53.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:54.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:51:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:51:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:54.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:54.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:54.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:51:54.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:54.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:54.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:51:54.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:51:54.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:54.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:51:54.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:54.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:54.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:51:54.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:51:54.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:54.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:51:54.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:54.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:54.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:51:54.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:51:54.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2504 2024-09-18T00:51:55.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2505 2024-09-18T00:51:55.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:55.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:56.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2506 2024-09-18T00:51:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2507 2024-09-18T00:51:56.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:56.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:51:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:51:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:51:56.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2509 2024-09-18T00:51:56.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:56.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:51:56.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:51:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:51:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2510 2024-09-18T00:51:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:51:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:51:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:51:57.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2508 2024-09-18T00:51:57.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:57.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:58.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:51:58.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:58.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:58.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:51:58.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:51:58.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:58.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:51:58.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:58.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:58.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:51:58.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:51:59.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2512 2024-09-18T00:51:59.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:59.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:51:59.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:59.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:59.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:51:59.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:51:59.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:51:59.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:51:59.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:51:59.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:51:59.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:51:59.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:51:59.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2513 2024-09-18T00:51:59.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2511 2024-09-18T00:51:59.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:59.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:51:59.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:00.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:00.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:52:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:52:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:52:00.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2514 2024-09-18T00:52:00.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2515 2024-09-18T00:52:00.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:00.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:52:00.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:00.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:00.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:52:00.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:52:00.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:00.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:52:00.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:00.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:00.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:52:00.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:52:00.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:01.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2517 2024-09-18T00:52:01.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2516 2024-09-18T00:52:01.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:01.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:02.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2519 2024-09-18T00:52:02.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2518 2024-09-18T00:52:02.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:02.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2520 2024-09-18T00:52:02.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:02.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:02.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:52:02.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:02.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:02.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:52:02.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:52:02.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:02.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:52:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:52:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:52:02.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:02.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:52:02.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:02.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:02.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:52:02.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:52:03.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:03.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:52:03.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:03.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:03.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:52:03.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:52:03.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:03.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:52:03.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:03.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:03.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:52:03.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:52:03.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2522 2024-09-18T00:52:03.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:03.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2521 2024-09-18T00:52:03.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2523 2024-09-18T00:52:03.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:52:04.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:04.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:04.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:52:04.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:52:04.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:04.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:52:04.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:04.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:04.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:52:04.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:52:05.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:05.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:52:05.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:05.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:05.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:52:05.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:52:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2525 2024-09-18T00:52:05.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:05.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:52:05.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:05.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:05.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:52:05.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:52:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2524 2024-09-18T00:52:05.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:05.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:52:05.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:05.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:05.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:52:05.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:52:05.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:06.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2526 2024-09-18T00:52:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2527 2024-09-18T00:52:06.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:06.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:06.766 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:52:06.760+0000 7fac32fac640 -1 reset not still connected to 0x55a87e1f1d40 2024-09-18T00:52:06.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:06.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:52:06.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:06.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:06.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:52:06.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:52:06.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:06.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:52:06.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:06.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:06.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:52:06.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:52:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:52:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:06.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:06.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:52:06.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:52:07.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2528 2024-09-18T00:52:07.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:07.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2529 2024-09-18T00:52:07.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2530 2024-09-18T00:52:07.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:07.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:08.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:08.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:52:08.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:08.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:08.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:52:08.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:52:08.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:08.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:52:08.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:08.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:08.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:52:08.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:52:08.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2532 2024-09-18T00:52:08.895 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:52:08.888+0000 7fac32fac640 -1 reset not still connected to 0x55a875e0a1a0 2024-09-18T00:52:08.895 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:52:08.888+0000 7fac32fac640 -1 reset not still connected to 0x55a87e1f1d40 2024-09-18T00:52:08.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2531 2024-09-18T00:52:08.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2533 2024-09-18T00:52:08.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:08.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:08.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:09.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:09.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:52:09.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:09.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:09.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:52:09.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:52:09.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:52:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:52:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:52:09.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2535 2024-09-18T00:52:09.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2534 2024-09-18T00:52:09.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:09.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:10.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:10.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:52:10.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:10.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:10.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:52:10.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:52:10.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:10.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:52:10.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:10.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:10.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:52:10.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:52:10.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:52:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:52:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:52:10.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2537 2024-09-18T00:52:10.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2536 2024-09-18T00:52:10.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:10.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2539 2024-09-18T00:52:11.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:11.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:52:11.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:11.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:11.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:52:11.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:52:11.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:11.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:11.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:52:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:52:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:52:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:52:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:52:11.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:52:11.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2540 2024-09-18T00:52:11.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:11.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2538 2024-09-18T00:52:11.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:12.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:12.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:52:12.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:12.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:12.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:52:12.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:52:12.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:12.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:52:12.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:12.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:52:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:52:13.830 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:52:13.824+0000 7fac32fac640 -1 reset not still connected to 0x55a87e1f1d40 2024-09-18T00:52:13.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2542 2024-09-18T00:52:13.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2541 2024-09-18T00:52:13.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:13.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2543 2024-09-18T00:52:13.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:13.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:13.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:13.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:52:13.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:13.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:13.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:52:13.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:52:13.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:13.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:52:13.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:13.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:13.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:52:13.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:52:14.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2544 2024-09-18T00:52:14.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:52:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:52:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:52:14.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2545 2024-09-18T00:52:14.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:14.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:14.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:14.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:52:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:52:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:52:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:52:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:14.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:52:14.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:52:15.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2547 2024-09-18T00:52:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2546 2024-09-18T00:52:15.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:15.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:16.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:16.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:52:16.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:16.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:16.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:52:16.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:52:16.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:16.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:52:16.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:16.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:16.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:52:16.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:52:16.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2548 2024-09-18T00:52:16.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2550 2024-09-18T00:52:16.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:16.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:52:16.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:16.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:16.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:52:16.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:52:16.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:16.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2549 2024-09-18T00:52:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:16.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:17.697 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:17.696+0000 7f52d7c6c640 -1 reset not still connected to 0x5578426955f0 2024-09-18T00:52:17.697 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:17.696+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f85b0 2024-09-18T00:52:17.698 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:17.696+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f89c0 2024-09-18T00:52:17.698 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:17.696+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T00:52:17.698 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:17.696+0000 7f52d7c6c640 -1 reset not still connected to 0x557844c39c70 2024-09-18T00:52:17.699 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:17.696+0000 7f52d7c6c640 -1 reset not still connected to 0x5578469ea0d0 2024-09-18T00:52:17.699 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:17.696+0000 7f52d7c6c640 -1 reset not still connected to 0x557846a424e0 2024-09-18T00:52:17.710 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:17.708+0000 7f52d7c6c640 -1 reset not still connected to 0x5578470b6340 2024-09-18T00:52:17.711 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:17.708+0000 7f52d7c6c640 -1 reset not still connected to 0x557848aa9ba0 2024-09-18T00:52:17.711 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:17.708+0000 7f52d7c6c640 -1 reset not still connected to 0x55784a12a9c0 2024-09-18T00:52:17.711 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:17.708+0000 7f52d7c6c640 -1 reset not still connected to 0x557872bb4dd0 2024-09-18T00:52:17.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:17.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:52:17.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:17.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:17.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:52:17.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:52:17.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:17.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:52:17.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:17.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:17.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:52:17.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:52:18.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:18.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:52:18.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:18.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:18.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:52:18.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:52:18.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2553 2024-09-18T00:52:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2551 2024-09-18T00:52:18.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2552 2024-09-18T00:52:18.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:18.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:52:18.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:18.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:18.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:52:18.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:52:19.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:19.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:19.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:19.938 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2555 2024-09-18T00:52:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:52:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:19.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:52:19.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:52:19.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2554 2024-09-18T00:52:19.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:52:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:52:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:52:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:19.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:19.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:52:19.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:19.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:19.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:52:19.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:52:19.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:20.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2556 2024-09-18T00:52:20.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2557 2024-09-18T00:52:21.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:21.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:21.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2558 2024-09-18T00:52:21.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:21.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2559 2024-09-18T00:52:21.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:52:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:52:22.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:52:22.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2560 2024-09-18T00:52:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:52:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:52:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:52:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:52:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:52:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:52:22.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:52:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:22.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:52:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:52:22.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:22.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:52:22.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:22.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:22.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:52:22.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:52:23.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2562 2024-09-18T00:52:23.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2561 2024-09-18T00:52:23.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:23.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2563 2024-09-18T00:52:24.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:24.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:52:24.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:24.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:24.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:52:24.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:52:24.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:24.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:52:24.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:24.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:24.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:52:24.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:52:24.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:24.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:24.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:24.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:52:24.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:24.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:24.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:52:24.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:52:24.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:24.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:52:24.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:24.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:24.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:52:24.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:52:25.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2564 2024-09-18T00:52:25.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2565 2024-09-18T00:52:25.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:25.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:25.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:25.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:52:25.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:25.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:25.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:52:25.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:52:26.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2566 2024-09-18T00:52:26.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2567 2024-09-18T00:52:26.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:26.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:26.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2569 2024-09-18T00:52:26.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:26.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:52:26.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:26.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:26.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:52:26.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:52:26.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2568 2024-09-18T00:52:27.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:52:27.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:52:27.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2570 2024-09-18T00:52:27.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:27.042 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:52:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:52:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:52:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:52:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:28.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:52:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:52:29.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2572 2024-09-18T00:52:29.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2571 2024-09-18T00:52:29.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2573 2024-09-18T00:52:29.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:29.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:52:29.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:52:29.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:52:29.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:52:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:52:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:52:29.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:29.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:30.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:30.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:52:30.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:30.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:30.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:52:30.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:52:30.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2574 2024-09-18T00:52:30.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:30.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2575 2024-09-18T00:52:30.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:30.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:52:30.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:30.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:30.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:52:30.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:52:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:52:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:52:30.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:52:30.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:30.743 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:52:30.739+0000 7f849c00d640 -1 reset not still connected to 0x5629c3616820 2024-09-18T00:52:31.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2577 2024-09-18T00:52:31.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2576 2024-09-18T00:52:31.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:31.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:31.851 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:31.843+0000 7f52d7c6c640 -1 reset not still connected to 0x5578469ea0d0 2024-09-18T00:52:31.851 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:31.847+0000 7f52d7c6c640 -1 reset not still connected to 0x557848aa9ba0 2024-09-18T00:52:31.855 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:31.855+0000 7f52d7c6c640 -1 reset not still connected to 0x55784a12a9c0 2024-09-18T00:52:32.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2580 2024-09-18T00:52:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:52:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:52:32.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:52:32.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2578 2024-09-18T00:52:32.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2579 2024-09-18T00:52:32.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:32.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:52:32.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:32.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:32.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:52:32.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:52:32.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:52:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:52:32.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:52:32.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:32.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:33.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:33.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:52:33.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:33.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:33.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:52:33.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:52:33.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:33.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:52:33.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:33.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:33.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:52:33.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:52:33.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2581 2024-09-18T00:52:33.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2582 2024-09-18T00:52:33.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:33.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:33.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2583 2024-09-18T00:52:33.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:34.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:34.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:52:34.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:34.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:34.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:52:34.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:52:34.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:34.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:52:34.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:34.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:34.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:52:34.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:52:35.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2584 2024-09-18T00:52:35.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2585 2024-09-18T00:52:35.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:35.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:52:35.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:35.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:35.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:52:35.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:35.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:35.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:52:35.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:35.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:35.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:52:35.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:52:35.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:52:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:35.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:35.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:52:35.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:52:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2587 2024-09-18T00:52:36.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2586 2024-09-18T00:52:36.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:36.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:52:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:36.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:52:36.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:52:36.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:36.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:52:36.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:36.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:36.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:52:36.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:52:36.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:36.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:52:36.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:36.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:36.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:52:36.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:52:37.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2588 2024-09-18T00:52:37.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2590 2024-09-18T00:52:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2589 2024-09-18T00:52:37.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:37.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:37.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:52:38.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:38.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:38.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:52:38.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:52:38.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:38.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:52:38.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:38.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:38.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:52:38.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:52:38.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2593 2024-09-18T00:52:38.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2592 2024-09-18T00:52:38.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2591 2024-09-18T00:52:38.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:38.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:38.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:39.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:39.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:52:39.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:39.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:39.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:52:39.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:52:39.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:39.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:52:39.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:39.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:39.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:52:39.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:52:39.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2594 2024-09-18T00:52:39.904 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2595 2024-09-18T00:52:39.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:39.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:52:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:52:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:52:40.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:40.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:52:40.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:40.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:40.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:52:40.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:52:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:52:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:52:40.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:52:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2597 2024-09-18T00:52:40.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2596 2024-09-18T00:52:40.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:40.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:41.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2598 2024-09-18T00:52:41.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2599 2024-09-18T00:52:41.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2600 2024-09-18T00:52:41.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:41.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:41.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:52:41.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:41.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:41.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:52:41.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:52:41.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:41.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:41.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:52:41.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:41.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:41.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:52:41.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:52:41.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:41.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:52:41.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:41.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:41.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:52:41.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:52:41.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:42.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:42.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:52:42.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:42.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:42.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:52:42.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:52:42.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:42.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:52:42.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:42.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:42.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:52:42.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:52:43.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2601 2024-09-18T00:52:43.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2602 2024-09-18T00:52:43.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:52:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:52:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:52:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:52:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:52:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:52:43.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:43.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2603 2024-09-18T00:52:43.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:43.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:44.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:44.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:52:44.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:44.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:44.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:52:44.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:52:44.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:44.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:52:44.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:44.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:44.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:52:44.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:52:44.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2605 2024-09-18T00:52:44.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2604 2024-09-18T00:52:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:52:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:52:44.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:52:44.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:44.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:45.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2607 2024-09-18T00:52:45.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2606 2024-09-18T00:52:45.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:45.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:46.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2610 2024-09-18T00:52:46.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2609 2024-09-18T00:52:46.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2608 2024-09-18T00:52:46.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:46.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:52:46.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:46.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:46.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:52:46.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:52:46.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:52:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:46.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:46.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:52:46.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:52:46.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:46.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:46.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:46.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:52:46.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:46.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:46.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:52:46.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:52:47.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:47.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:52:47.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:47.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:47.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:52:47.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:52:47.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:52:47.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 108714 2024-09-18T00:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2613 2024-09-18T00:52:48.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:52:48.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 108847 2024-09-18T00:52:48.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2611 2024-09-18T00:52:48.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:48.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:52:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:52:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:52:48.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2612 2024-09-18T00:52:48.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:49.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:49.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:49.665 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-18T00:52:49.663+0000 7fbbe8b19640 -1 reset not still connected to 0x561a1fd885b0 2024-09-18T00:52:49.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2614 2024-09-18T00:52:49.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:52:49.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 109001 2024-09-18T00:52:49.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:52:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:52:49.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:49.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:52:49.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:52:49.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:49.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:49.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:52:49.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:52:50.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2615 2024-09-18T00:52:51.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:51.642 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:52:51.635+0000 7f52d7c6c640 -1 reset not still connected to 0x557872bb4dd0 2024-09-18T00:52:51.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2617 2024-09-18T00:52:51.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2616 2024-09-18T00:52:51.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:51.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:52.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:52.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:52.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:52:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:52:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:52:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:52:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:52:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:52:52.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:52:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:52:52.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:52:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:52:52.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 109173 2024-09-18T00:52:53.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2618 2024-09-18T00:52:54.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:54.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:52:54.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:54.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:54.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:52:54.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:52:54.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:54.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2619 2024-09-18T00:52:54.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2620 2024-09-18T00:52:54.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:54.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:54.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:52:54.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:54.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:54.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:52:54.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:52:55.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:52:55.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 109313 2024-09-18T00:52:56.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2621 2024-09-18T00:52:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:52:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:52:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:52:57.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2622 2024-09-18T00:52:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:52:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:52:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:52:57.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:52:57.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:52:57.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:52:57.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:52:57.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 109580 2024-09-18T00:52:57.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:59.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2624 2024-09-18T00:52:59.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2623 2024-09-18T00:52:59.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:52:59.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 109763 2024-09-18T00:52:59.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:52:59.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:53:00.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:53:00.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:53:00.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:53:00.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:53:00.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:53:00.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:53:02.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2625 2024-09-18T00:53:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:53:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 109937 2024-09-18T00:53:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:53:02.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 110197 2024-09-18T00:53:02.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:53:05.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:53:05.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:53:05.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 108582 2024-09-18T00:55:19.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:55:19.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:55:19.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:55:19.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 108715 2024-09-18T00:55:20.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:55:20.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:55:20.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:55:20.793 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 108848 2024-09-18T00:55:22.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:55:22.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:55:22.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:55:22.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 109002 2024-09-18T00:55:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:55:23.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:55:23.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:55:23.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 109174 2024-09-18T00:55:26.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:55:26.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:55:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:55:26.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 109314 2024-09-18T00:55:27.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:55:27.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:55:27.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:55:27.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 109581 2024-09-18T00:55:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:55:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:55:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:55:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 109764 2024-09-18T00:55:32.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:55:32.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:55:32.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:55:32.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 109938 2024-09-18T00:55:33.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:55:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:55:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T00:55:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 110198 2024-09-18T00:55:36.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T00:55:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T00:55:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:55:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:55:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:55:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:55:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 1 2024-09-18T00:55:36.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:55:36.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd0 2024-09-18T00:55:36.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary1 2024-09-18T00:55:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary1 2024-09-18T00:55:36.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:55:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:55:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary1 2024-09-18T00:55:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 2 2024-09-18T00:55:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:55:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:55:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd1 2024-09-18T00:55:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 3 2024-09-18T00:55:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary2 2024-09-18T00:55:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary2 2024-09-18T00:55:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:55:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary2 2024-09-18T00:55:36.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd2 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary3 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary3 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary3 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 4 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd3 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 5 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary4 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary4 2024-09-18T00:55:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:55:36.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:55:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary4 2024-09-18T00:55:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd4 2024-09-18T00:55:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary5 2024-09-18T00:55:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary5 2024-09-18T00:55:36.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:55:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary5 2024-09-18T00:55:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:55:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:55:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 6 2024-09-18T00:55:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd5 2024-09-18T00:55:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary6 2024-09-18T00:55:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary6 2024-09-18T00:55:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:55:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary6 2024-09-18T00:55:36.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 7 2024-09-18T00:55:36.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:55:36.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:55:36.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd6 2024-09-18T00:55:36.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary7 2024-09-18T00:55:36.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary7 2024-09-18T00:55:36.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:55:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary7 2024-09-18T00:55:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:55:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:55:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 8 2024-09-18T00:55:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 9 2024-09-18T00:55:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:55:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd7 2024-09-18T00:55:36.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:55:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary8 2024-09-18T00:55:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary8 2024-09-18T00:55:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:55:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd8 2024-09-18T00:55:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary8 2024-09-18T00:55:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary9 2024-09-18T00:55:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary9 2024-09-18T00:55:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:55:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary9 2024-09-18T00:55:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 10 2024-09-18T00:55:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T00:55:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd9 2024-09-18T00:55:36.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary10 2024-09-18T00:55:36.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary10 2024-09-18T00:55:36.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:55:36.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T00:55:36.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary10 2024-09-18T00:55:36.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 116069 2024-09-18T00:55:36.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:55:37.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:55:37.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:55:37.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:55:37.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:55:37.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:55:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:55:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:55:37.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:55:37.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:55:37.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:55:37.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:55:37.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:55:37.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:55:37.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:55:37.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:55:37.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:55:37.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:55:37.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:55:37.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:56:35.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=bdd796b452f0468d0766d734471772c1 2024-09-18T00:56:35.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary7 2024-09-18T00:56:36.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary7 2024-09-18T00:56:36.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:36.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:36.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:56:36.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary7 2024-09-18T00:56:37.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=04243d485e139a51e9a172fb4074fdd1 2024-09-18T00:56:37.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary8 2024-09-18T00:56:38.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=6ea634d7a28a3675bd843f7c5be12451 2024-09-18T00:56:38.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary4 2024-09-18T00:56:38.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=c15df145e5fd52270e39b1434ed51e66 2024-09-18T00:56:38.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary5 2024-09-18T00:56:38.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:56:38.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+unknown 2024-09-18T00:56:38.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:38.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:38.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:56:38.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:38.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:38.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:38.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:38.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:38.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary8 2024-09-18T00:56:38.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:56:38.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary8 2024-09-18T00:56:38.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=dbc2664576ad7c5b66c0aa4981af740a 2024-09-18T00:56:38.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary6 2024-09-18T00:56:38.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=440df9124bc5666807357f89d11ab20a 2024-09-18T00:56:38.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary3 2024-09-18T00:56:39.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=674b49c6cf93f9ebbe5fa35ae7ecd657 2024-09-18T00:56:39.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary1 2024-09-18T00:56:39.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=fc87f6ac90dfe0071af90778d6935192 2024-09-18T00:56:39.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary9 2024-09-18T00:56:39.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary4 2024-09-18T00:56:39.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:39.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:39.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:56:39.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary4 2024-09-18T00:56:39.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary5 2024-09-18T00:56:39.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:39.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:39.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:56:39.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary5 2024-09-18T00:56:39.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=1cff2b1829b110e48d6e472e435149a1 2024-09-18T00:56:39.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary10 2024-09-18T00:56:39.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T00:56:39.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:39.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:39.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:56:39.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:39.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:39.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:39.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:39.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:39.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:56:39.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:56:39.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 682d0054-0ced-419a-bc1c-f9e86132b9fc 2024-09-18T00:56:39.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:39.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:39.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:39.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:39.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:39.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:39.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:39.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:39.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39' 2024-09-18T00:56:39.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary6 2024-09-18T00:56:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:56:39.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary6 2024-09-18T00:56:39.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary3 2024-09-18T00:56:39.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:39.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:39.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:56:39.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary3 2024-09-18T00:56:39.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary1 2024-09-18T00:56:39.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:39.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:39.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:56:39.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary1 2024-09-18T00:56:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:39.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:39.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:39.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:39.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:39.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:39.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:39.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:39' 2024-09-18T00:56:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+unknown '2024-09-18 00:56:39' 2024-09-18T00:56:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:56:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:39' 2024-09-18T00:56:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:40.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:40.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:40.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary9 2024-09-18T00:56:40.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:40.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:40.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:56:40.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary9 2024-09-18T00:56:40.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2172928.0,"last_snapshot_bytes":2654208,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620763,"remote_snapshot_timestamp":1726620763,"replay_state":"idle"}2024-09-18 00:56:39image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":5283840.0,"last_snapshot_bytes":5906432,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620780,"remote_snapshot_timestamp":1726620780,"replay_state":"idle"}2024-09-18 00:56:39image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":952320.0,"last_snapshot_bytes":1703936,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620764,"remote_snapshot_timestamp":1726620764,"replay_state":"idle"}2024-09-18 00:56:39image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1701888.0,"last_snapshot_bytes":1601536,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620765,"remote_snapshot_timestamp":1726620765,"replay_state":"idle"}2024-09-18 00:56:39image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2619392.0,"last_snapshot_bytes":229376,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620768,"remote_snapshot_timestamp":1726620768,"replay_state":"idle"}2024-09-18 00:56:39image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2641920.0,"last_snapshot_bytes":5128192,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620770,"remote_snapshot_timestamp":1726620770,"replay_state":"idle"}2024-09-18 00:56:39image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":178176.0,"last_snapshot_bytes":135168,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620772,"remote_snapshot_timestamp":1726620772,"replay_state":"idle"}2024-09-18 00:56:39image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:39image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2146304.0,"last_snapshot_bytes":2564096,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620777,"remote_snapshot_timestamp":1726620777,"replay_state":"idle"}2024-09-18 00:56:39image-primary993ec4895-421e-451c-941b-0c6809cf872eup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 00:56:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2619392.0,"last_snapshot_bytes":233472,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620777,"remote_snapshot_timestamp":1726620777,"replay_state":"idle"}2024-09-18 00:56:39' 2024-09-18T00:56:40.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:40.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:56:40.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary10 2024-09-18T00:56:40.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:40.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:40.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:56:40.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary10 2024-09-18T00:56:40.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:56:40.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+unknown 2024-09-18T00:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:40.588 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:56:40.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+unknown 2024-09-18T00:56:40.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:40.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:40.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:56:40.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:40.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:40.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:40.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:40.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+unknown 2024-09-18T00:56:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:56:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:40.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:40.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:40.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:40.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:41.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=92422203fde55f99b33a5c3c1eaae4af 2024-09-18T00:56:41.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary2 2024-09-18T00:56:41.072 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:56:41.065+0000 7f52d7c6c640 -1 reset not still connected to 0x55784a12a9c0 2024-09-18T00:56:41.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:39' 2024-09-18T00:56:41.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:56:41.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:41.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:41.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary7 up+unknown 2024-09-18T00:56:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:56:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:41.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:41.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:56:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+unknown 2024-09-18T00:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:41.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:56:41.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:41.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:41.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:56:41.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:41.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:41.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:41.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:41.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:41.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:56:41.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:56:41.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+unknown 2024-09-18T00:56:41.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:41.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:41.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:56:41.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:41.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:41.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:41.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:41.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:41.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:56:41.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:41.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:41.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:56:41.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:41.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:41.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:41.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:41.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:41.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:56:41.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:56:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:56:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:41.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:56:41.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:41.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:41.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:41.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:41.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:41.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:56:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+unknown 2024-09-18T00:56:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:56:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:41.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+unknown 2024-09-18T00:56:41.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:41.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:41.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:56:41.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:41.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:41.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:41.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:41.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:41.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:56:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: c183a554-660d-4696-b42f-a3c4a75ef26e 2024-09-18T00:56:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:56:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:56:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:56:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:40' 2024-09-18T00:56:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:56:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:56:41.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:56:41.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: cd38d20d-4581-4726-a3c5-e2f4f19b40f5 2024-09-18T00:56:41.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:56:41.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:56:41.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:41.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:41.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:41.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:41.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:41.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:56:41.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:40' 2024-09-18T00:56:41.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:41.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:56:41.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:41.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:56:41.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:56:41.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8b737e05-c466-4ab1-8f79-d9dc4c934fea 2024-09-18T00:56:41.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:56:41.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:56:41.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:41.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:41.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:41.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:41.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:41.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:56:41.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:40' 2024-09-18T00:56:41.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:41.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:56:41.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:41.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:56:41.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary2 2024-09-18T00:56:41.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:41.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:41.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:56:41.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary2 2024-09-18T00:56:42.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary7 up+unknown '' '' 2024-09-18T00:56:42.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:42.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:42.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:56:42.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:42.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:42.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:42.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:42.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:42.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary7 2024-09-18T00:56:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:56:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 682d0054-0ced-419a-bc1c-f9e86132b9fc 2024-09-18T00:56:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:56:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39' 2024-09-18T00:56:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:42.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:42.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:42.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:42.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:42.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:39' 2024-09-18T00:56:42.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary7 up+unknown '2024-09-18 00:56:39' 2024-09-18T00:56:42.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:42.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:42.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:56:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:39' 2024-09-18T00:56:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:42.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:42.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:56:41cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:56:39image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":5283840.0,"last_snapshot_bytes":5906432,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620780,"remote_snapshot_timestamp":1726620780,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:56:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:56:39image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":952320.0,"last_snapshot_bytes":1703936,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726620764,"remote_snapshot_timestamp":1726620764,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 00:56:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:56:39image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:56:41cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:56:39image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:56:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:56:39image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:56:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:56:39image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:56:41cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:56:39image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:39image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:56:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:56:39image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 00:56:41cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 00:56:39' 2024-09-18T00:56:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:42.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:56:42.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:39' 2024-09-18T00:56:42.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:56:42.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:42.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:42.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:42.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:42.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:42.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:42.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary7 2024-09-18T00:56:42.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:42.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:42.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:56:42.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:56:42.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary7 2024-09-18T00:56:42.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:56:42.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+unknown 2024-09-18T00:56:42.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:56:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:42.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T00:56:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:56:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:42.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:56:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T00:56:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:56:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:42.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:42.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:42.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:56:42.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:56:42.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:42.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:42.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:56:42.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:42.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:42.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:42.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:42.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:42.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:56:42.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T00:56:42.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:42.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:42.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:56:42.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:42.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:42.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:42.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:42.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:42.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:56:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:56:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 238293d6-165c-443a-acb3-b579f41e40a2 2024-09-18T00:56:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:56:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:56:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:56:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:41' 2024-09-18T00:56:42.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:56:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:56:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:56:42.686 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 77b516d0-a26f-44c3-b75d-8945de8e9052 2024-09-18T00:56:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:56:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:56:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:56:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:41' 2024-09-18T00:56:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:56:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:56:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:56:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9a782482-1483-42e7-a43c-95bec4be0376 2024-09-18T00:56:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:56:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:56:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:56:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:41' 2024-09-18T00:56:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:56:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:42.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:56:42.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:56:42.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 93ec4895-421e-451c-941b-0c6809cf872e 2024-09-18T00:56:42.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:56:42.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:56:42.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:42.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:42.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:42.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:42.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:42.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:56:42.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:41' 2024-09-18T00:56:42.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:42.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:56:42.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:42.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:56:42.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:56:42.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:56:42.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:56:43.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd0 2024-09-18T00:56:43.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T00:56:43.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:56:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T00:56:43.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+unknown 2024-09-18T00:56:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:56:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:43.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T00:56:43.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:43.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:43.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:56:43.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:43.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:43.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:43.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:43.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:43.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:56:43.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:56:43.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 6f07d2c4-2b18-4dec-be82-bb3768331b71 2024-09-18T00:56:43.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:56:43.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:56:43.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:43.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:43.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:43.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:43.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:43.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:56:43.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:42' 2024-09-18T00:56:43.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:43.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:56:43.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:43.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T00:56:43.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:56:43.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:43.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:43.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:56:43.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:43.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:43.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:43.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:43.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:43.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:56:43.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:56:43.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:43.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:43.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:56:43.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:43.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:43.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:43.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:43.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:43.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:56:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:56:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:56:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:56:43.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:56:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: c183a554-660d-4696-b42f-a3c4a75ef26e 2024-09-18T00:56:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:56:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:56:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:56:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:40' 2024-09-18T00:56:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:43.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:56:43.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:43.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:56:43.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:56:43.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: cd38d20d-4581-4726-a3c5-e2f4f19b40f5 2024-09-18T00:56:43.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:56:43.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:56:43.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:43.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:43.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:43.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:43.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:40' 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8b737e05-c466-4ab1-8f79-d9dc4c934fea 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:39 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T00:56:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:40' 2024-09-18T00:56:43.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:43.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:56:43.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T00:56:43.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:43.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:56:43.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:43.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T00:56:44.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T00:56:44.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:44.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:44.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:56:44.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:44.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:44.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:44.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:44.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:44.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:56:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T00:56:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:56:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:44.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:44.666 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:56:44.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T00:56:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:56:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:56:44.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T00:56:44.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:44.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:44.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:56:44.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:44.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:44.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:44.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:56:44.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:56:44.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a96adece-8cff-4940-898a-d2a762b46aea 2024-09-18T00:56:44.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:44.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:44.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:44.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:44.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:44.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:44.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:44.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T00:56:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:56:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:44.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:44.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:44.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:44.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:56:44.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:44.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:44.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:44.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:44.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:44.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:44.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:44.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:56:44.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:44.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:44.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:44.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:44.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:44.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:56:44.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 238293d6-165c-443a-acb3-b579f41e40a2 2024-09-18T00:56:44.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:44.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:44.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:44.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:44.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:44.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:44.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:44.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:44.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:44.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:44.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:44.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:44.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:44.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:44.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:44.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:44.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:44.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:44.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:44.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:44.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:56:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 77b516d0-a26f-44c3-b75d-8945de8e9052 2024-09-18T00:56:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:44.789 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:44.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:44.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:44.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:44.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:44.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:44.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:44.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:44.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:44.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:44.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:44.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:44.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:44.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:44.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:44.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:44.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:56:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:44.796 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:44.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:44.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:44.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:44.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:56:44.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9a782482-1483-42e7-a43c-95bec4be0376 2024-09-18T00:56:44.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:44.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:56:44.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:44.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:44.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:44.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:44.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:44.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:44.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:44.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:44.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:44.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:44.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:44.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:44.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:44.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:44.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:56:44.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 93ec4895-421e-451c-941b-0c6809cf872e 2024-09-18T00:56:44.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:44.814 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:44.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:44.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:44.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:44.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:44.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:44.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:44.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:44.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:44.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:44.816 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:44.817 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:44.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:44.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:44.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:44.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:44.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:44.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:44.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:44.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:44.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:56:44.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:44.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:44.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:44.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:44.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:44.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:44.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:44.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:44.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:44.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:56:44.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:44.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:44.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:44.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:44.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:44.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:44.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:44.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:56:44.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:44.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:56:44.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:44.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:44.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:44.876 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary2 up+unknown 2024-09-18T00:56:44.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:56:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:44.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:44.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:56:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:56:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:44.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:44.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:44.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:44.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary3 up+unknown 2024-09-18T00:56:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:56:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:45.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:45.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:45.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:45.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:45.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:56:45.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:56:45.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:45.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:45.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:56:45.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:56:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:45.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:45.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:45.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:45.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary6 up+unknown 2024-09-18T00:56:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary9 up+unknown 2024-09-18T00:56:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:56:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:56:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:45.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:56:45.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:45.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:56:45.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:45.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:45.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:45.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:45.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:45.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:45.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary1 up+unknown 2024-09-18T00:56:45.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:45.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:45.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:56:45.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:45.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:45.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:45.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:45.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:45.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T00:56:45.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:45.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:45.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:56:45.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:56:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:56:45.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 6f07d2c4-2b18-4dec-be82-bb3768331b71 2024-09-18T00:56:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:45.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:45.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:45.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:45.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:45.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:45.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:45.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:45.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:45.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:45.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:45.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:45.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:45.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:45.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:45.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:45.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:45.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:56:45.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:45.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:45.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:45.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:45.751 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:45.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary2 up+unknown '' '' 2024-09-18T00:56:45.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:45.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:45.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:56:45.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:45.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:45.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:45.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:45.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:45.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary2 2024-09-18T00:56:45.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:45.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:45.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:56:45.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:45.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:56:45.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:45.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:45.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:45.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:45.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:45.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:45.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary10 up+unknown 2024-09-18T00:56:45.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:45.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:45.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:56:45.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:45.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:45.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:45.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:45.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:56:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a96adece-8cff-4940-898a-d2a762b46aea 2024-09-18T00:56:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:45.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:56:45.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:45.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:45.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:45.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:45.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:45.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:45.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:45.947 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:45.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:45.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:45.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:45.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary2 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:45.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:45.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:56:45.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:45.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:45.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:45.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:45.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary3 up+unknown '' '' 2024-09-18T00:56:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:56:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:45.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:45.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary3 2024-09-18T00:56:46.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary9 up+unknown '' '' 2024-09-18T00:56:46.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:56:46.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:46.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:46.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:46.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:46.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary6 up+unknown '' '' 2024-09-18T00:56:46.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:56:46.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:46.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:46.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:46.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:46.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:46.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:46.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary6 2024-09-18T00:56:46.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary9 2024-09-18T00:56:46.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary1 up+unknown '' '' 2024-09-18T00:56:46.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:56:46.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:46.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:46.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:46.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:46.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:46.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary1 2024-09-18T00:56:46.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:56:46.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 238293d6-165c-443a-acb3-b579f41e40a2 2024-09-18T00:56:46.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:46.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:46.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:46.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:46.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:46.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:56:46.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:46.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:46.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:46.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:46.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:46.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:46.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:46.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:46.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:46.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:46.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:46.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:46.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:46.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:46.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:56:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:46.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:46.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:46.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary2 2024-09-18T00:56:46.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:56:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:56:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary2 2024-09-18T00:56:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary3 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:56:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:46.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:46.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:46.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:46.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:56:46.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 77b516d0-a26f-44c3-b75d-8945de8e9052 2024-09-18T00:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:46.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:46.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:46.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:46.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:46.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:46.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:46.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:46.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:46.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:46.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:46.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:46.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:46.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:46.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary6 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:46.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:56:46.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:46.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:46.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:46.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:46.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:46.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:56:46.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9a782482-1483-42e7-a43c-95bec4be0376 2024-09-18T00:56:46.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:46.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:46.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:46.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:46.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:46.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:56:46.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:46.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:56:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 93ec4895-421e-451c-941b-0c6809cf872e 2024-09-18T00:56:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:56:46.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:46.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:46.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:46.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:46.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:46.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:46.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:46.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:46.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:46.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:46.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:46.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:46.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:46.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:46.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:46.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:46.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:46.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:46.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:46.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:46.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:46.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:46.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:46.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:46.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:46.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:46.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:46.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary1 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:46.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:56:46.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:46.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:46.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:46.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:46.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:46.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:46.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary9 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:46.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:56:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:46.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:46.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:56:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:46.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:56:46.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:46.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:46.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:46.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:46.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:46.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:46.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary3 2024-09-18T00:56:46.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:56:46.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:56:46.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary3 2024-09-18T00:56:46.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:46.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:56:46.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:46.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:56:46.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:46.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:46.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:46.417 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:46.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:46.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:46.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary9 2024-09-18T00:56:46.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:56:46.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:56:46.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary9 2024-09-18T00:56:46.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:46.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:46.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:56:46.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:46.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:46.423 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:56:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:46.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:56:46.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:46.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:56:46.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:46.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:46.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:46.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:46.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:46.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:46.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary6 2024-09-18T00:56:46.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:56:46.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:56:46.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary6 2024-09-18T00:56:46.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:46.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:46.433 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:46.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:46.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:46.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:46.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary1 2024-09-18T00:56:46.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:56:46.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:56:46.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary1 2024-09-18T00:56:46.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:56:46.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:56:46.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:56:46.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:56:46.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:56:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:56:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:56:46.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:56:46.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:56:46.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:56:46.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:56:46.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:56:46.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:56:46.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:56:46.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:56:46.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd1 2024-09-18T00:56:46.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T00:56:46.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:56:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd2 2024-09-18T00:56:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T00:56:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:56:46.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd3 2024-09-18T00:56:46.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T00:56:46.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:56:46.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary10 up+unknown '' '' 2024-09-18T00:56:46.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:46.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:46.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:56:46.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:46.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:46.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:46.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:46.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:46.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary10 2024-09-18T00:56:46.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd4 2024-09-18T00:56:46.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T00:56:46.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:56:46.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd5 2024-09-18T00:56:46.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T00:56:46.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:56:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:56:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 6f07d2c4-2b18-4dec-be82-bb3768331b71 2024-09-18T00:56:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:56:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:47.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:47.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:47.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:47.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:47.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:47.015 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:47.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:47.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:47.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary10 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:56:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:47.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:47.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:47.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:56:47.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:47.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:56:47.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:47.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:47.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:47.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:47.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:47.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:47.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary10 2024-09-18T00:56:47.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:47.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:47.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:56:47.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:56:47.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary10 2024-09-18T00:56:47.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:56:47.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:56:47.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:56:47.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T00:56:47.770 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:47.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:47.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:56:47.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:47.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:47.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:47.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:47.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:47.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:56:47.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T00:56:47.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:47.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:47.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:56:47.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:47.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:47.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:47.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:47.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T00:56:47.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:47.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:47.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:56:47.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:47.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:47.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:47.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:47.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:47.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:56:47.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:47.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:56:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd6 2024-09-18T00:56:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:56:47.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T00:56:47.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:56:47.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: c183a554-660d-4696-b42f-a3c4a75ef26e 2024-09-18T00:56:47.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:47.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:47.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:47.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:47.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8b737e05-c466-4ab1-8f79-d9dc4c934fea 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:47.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:47.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:47.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:47.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:47.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:47.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:47.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:47.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:47.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:47.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:47.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:47.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:48.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:48.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:48.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:48.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:56:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: cd38d20d-4581-4726-a3c5-e2f4f19b40f5 2024-09-18T00:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44 2024-09-18T00:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:44' 2024-09-18T00:56:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:48.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:48.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:48.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:48.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:48.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:48.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:48.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:48.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:48.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:48.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:48.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:48.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:48.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:48.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:48.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:48.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:56:48.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:48.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:48.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:48.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:48.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:48.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:48.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:48.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:48.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:48.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:48.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:48.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+unknown '2024-09-18 00:56:44' 2024-09-18T00:56:48.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:56:48.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:48.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:56:48.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:48.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:44' 2024-09-18T00:56:48.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:48.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:48.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:48.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:48.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:48.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:56:48.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:48.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:56:48.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:48.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:48.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:48.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:48.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:48.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:48.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary4 up+unknown 2024-09-18T00:56:48.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:48.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:48.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:56:48.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:48.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:48.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:48.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:48.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:48.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:48.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:48.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:56:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 00:56:44cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 00:56:44' 2024-09-18T00:56:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:48.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:56:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:48.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:56:48.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:44' 2024-09-18T00:56:48.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:56:48.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:48.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:48.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:48.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:48.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:48.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:48.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary5 up+unknown 2024-09-18T00:56:48.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:48.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:48.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:56:48.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:48.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:48.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:48.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:48.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:48.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:48.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:48.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:48.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:48.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:48.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:48.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary8 up+unknown 2024-09-18T00:56:48.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:48.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:48.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:56:48.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:48.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:48.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:48.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:56:48.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:56:49.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary4 up+unknown '' '' 2024-09-18T00:56:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:56:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:49.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:49.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary4 2024-09-18T00:56:49.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary5 up+unknown '' '' 2024-09-18T00:56:49.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:49.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:49.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:56:49.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:49.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:49.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:49.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary8 up+unknown '' '' 2024-09-18T00:56:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:49.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:56:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:56:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:56:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:49.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary8 2024-09-18T00:56:49.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:49.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary5 2024-09-18T00:56:49.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:56:49.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: cd38d20d-4581-4726-a3c5-e2f4f19b40f5 2024-09-18T00:56:49.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:49.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:49.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:49.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:49 2024-09-18T00:56:49.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:49.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:56:49.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:49.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:49.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:49' 2024-09-18T00:56:49.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:49.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:49.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:49.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:49.615 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:49.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:49.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:49.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:49.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:49.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:56:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: c183a554-660d-4696-b42f-a3c4a75ef26e 2024-09-18T00:56:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:49.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:49 2024-09-18T00:56:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:56:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:49' 2024-09-18T00:56:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:49.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:49.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:49.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:49.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:49' 2024-09-18T00:56:49.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary4 up+unknown '2024-09-18 00:56:49' 2024-09-18T00:56:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:56:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:49' 2024-09-18T00:56:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:49.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:49.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:49.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:49.658 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:49.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:49' 2024-09-18T00:56:49.668 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary8 up+unknown '2024-09-18 00:56:49' 2024-09-18T00:56:49.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:49.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:49.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:56:49.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:49.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:49' 2024-09-18T00:56:49.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:49.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:49.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:49.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:56:49.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8b737e05-c466-4ab1-8f79-d9dc4c934fea 2024-09-18T00:56:49.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:49.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:49.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:56:49.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:49 2024-09-18T00:56:49.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:56:49.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T00:56:49.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T00:56:49.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T00:56:49.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:56:49' 2024-09-18T00:56:49.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T00:56:49.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T00:56:49.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:56:49.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T00:56:49.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:56:49.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:49.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:56:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:49.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:56:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:56:49.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:56:49.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:56:49.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:49' 2024-09-18T00:56:49.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary5 up+unknown '2024-09-18 00:56:49' 2024-09-18T00:56:49.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:49.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:49.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:56:49.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T00:56:49.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:56:49' 2024-09-18T00:56:49.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:56:49.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:56:49.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T00:56:50.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}2024-09-18 00:56:49image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:49image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:49image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:49image-primary993ec4895-421e-451c-941b-0c6809cf872eup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49' 2024-09-18T00:56:50.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:50.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:56:50.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:49' 2024-09-18T00:56:50.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:56:50.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:50.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:50.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:50.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary8 2024-09-18T00:56:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:56:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:56:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary8 2024-09-18T00:56:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}2024-09-18 00:56:49image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:49image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:49image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:49image-primary993ec4895-421e-451c-941b-0c6809cf872eup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49' 2024-09-18T00:56:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:56:50.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:49' 2024-09-18T00:56:50.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:56:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:50.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:50.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary4 2024-09-18T00:56:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:50.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:56:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:56:50.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary4 2024-09-18T00:56:50.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}2024-09-18 00:56:49image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:49image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:49image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}2024-09-18 00:56:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 00:56:49image-primary993ec4895-421e-451c-941b-0c6809cf872eup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 00:56:49cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}2024-09-18 00:56:49' 2024-09-18T00:56:50.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:56:50.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:56:50.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:56:49' 2024-09-18T00:56:50.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:56:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T00:56:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T00:56:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T00:56:50.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T00:56:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:56:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary5 2024-09-18T00:56:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:56:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:56:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:56:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T00:56:50.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary5 2024-09-18T00:56:50.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:56:50.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:56:50.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:56:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd7 2024-09-18T00:56:51.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T00:56:51.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:56:51.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:56:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T00:56:51.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:56:51.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:56:51.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:56:51.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:56:52.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd8 2024-09-18T00:56:52.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:56:52.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T00:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd9 2024-09-18T00:56:52.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T00:56:52.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T00:57:09.344 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:57:09.344+0000 7fd4c158b640 -1 reset not still connected to 0x564358707450 2024-09-18T00:57:09.344 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:57:09.344+0000 7fd4c158b640 -1 reset not still connected to 0x5643587075f0 2024-09-18T00:57:09.344 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:57:09.344+0000 7fd4c158b640 -1 reset not still connected to 0x564358707d40 2024-09-18T00:57:09.345 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:57:09.344+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a340 2024-09-18T00:57:09.345 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:57:09.344+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T00:57:09.345 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T00:57:09.344+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T00:58:01.199 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:58:01.190+0000 7f52d7c6c640 -1 reset not still connected to 0x55784ea1fba0 2024-09-18T00:58:04.928 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T00:58:04.922+0000 7f52d7c6c640 -1 reset not still connected to 0x55784ea1fee0 2024-09-18T00:58:23.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=bdd796b452f0468d0766d734471772c1 2024-09-18T00:58:23.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd0 2024-09-18T00:58:24.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ bdd796b452f0468d0766d734471772c1 != \b\d\d\7\9\6\b\4\5\2\f\0\4\6\8\d\0\7\6\6\d\7\3\4\4\7\1\7\7\2\c\1 ]] 2024-09-18T00:58:33.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=92422203fde55f99b33a5c3c1eaae4af 2024-09-18T00:58:33.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd1 2024-09-18T00:58:34.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=fc87f6ac90dfe0071af90778d6935192 2024-09-18T00:58:34.669 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd4 2024-09-18T00:58:34.911 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 92422203fde55f99b33a5c3c1eaae4af != \9\2\4\2\2\2\0\3\f\d\e\5\5\f\9\9\b\3\3\a\5\c\3\c\1\e\a\a\e\4\a\f ]] 2024-09-18T00:58:35.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ fc87f6ac90dfe0071af90778d6935192 != \f\c\8\7\f\6\a\c\9\0\d\f\e\0\0\7\1\a\f\9\0\7\7\8\d\6\9\3\5\1\9\2 ]] 2024-09-18T00:58:36.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=674b49c6cf93f9ebbe5fa35ae7ecd657 2024-09-18T00:58:36.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd3 2024-09-18T00:58:36.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=dbc2664576ad7c5b66c0aa4981af740a 2024-09-18T00:58:36.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd5 2024-09-18T00:58:37.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 674b49c6cf93f9ebbe5fa35ae7ecd657 != \6\7\4\b\4\9\c\6\c\f\9\3\f\9\e\b\b\e\5\f\a\3\5\a\e\7\e\c\d\6\5\7 ]] 2024-09-18T00:58:37.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:58:37.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 116070 2024-09-18T00:58:37.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:58:37.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 116071 2024-09-18T00:58:37.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ dbc2664576ad7c5b66c0aa4981af740a != \d\b\c\2\6\6\4\5\7\6\a\d\7\c\5\b\6\6\c\0\a\a\4\9\8\1\a\f\7\4\0\a ]] 2024-09-18T00:58:37.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=1cff2b1829b110e48d6e472e435149a1 2024-09-18T00:58:37.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd6 2024-09-18T00:58:38.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=440df9124bc5666807357f89d11ab20a 2024-09-18T00:58:38.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd2 2024-09-18T00:58:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 1cff2b1829b110e48d6e472e435149a1 != \1\c\f\f\2\b\1\8\2\9\b\1\1\0\e\4\8\d\6\e\4\7\2\e\4\3\5\1\4\9\a\1 ]] 2024-09-18T00:58:38.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 440df9124bc5666807357f89d11ab20a != \4\4\0\d\f\9\1\2\4\b\c\5\6\6\6\8\0\7\3\5\7\f\8\9\d\1\1\a\b\2\0\a ]] 2024-09-18T00:58:38.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:58:38.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 116073 2024-09-18T00:58:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=04243d485e139a51e9a172fb4074fdd1 2024-09-18T00:58:39.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd7 2024-09-18T00:58:39.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=6ea634d7a28a3675bd843f7c5be12451 2024-09-18T00:58:39.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd9 2024-09-18T00:58:39.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=c15df145e5fd52270e39b1434ed51e66 2024-09-18T00:58:39.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd8 2024-09-18T00:58:40.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 04243d485e139a51e9a172fb4074fdd1 != \0\4\2\4\3\d\4\8\5\e\1\3\9\a\5\1\e\9\a\1\7\2\f\b\4\0\7\4\f\d\d\1 ]] 2024-09-18T00:58:40.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 6ea634d7a28a3675bd843f7c5be12451 != \6\e\a\6\3\4\d\7\a\2\8\a\3\6\7\5\b\d\8\4\3\f\7\c\5\b\e\1\2\4\5\1 ]] 2024-09-18T00:58:40.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:58:40.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 116076 2024-09-18T00:58:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ c15df145e5fd52270e39b1434ed51e66 != \c\1\5\d\f\1\4\5\e\5\f\d\5\2\2\7\0\e\3\9\b\1\4\3\4\e\d\5\1\e\6\6 ]] 2024-09-18T00:58:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:58:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 116078 2024-09-18T00:58:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:58:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 116080 2024-09-18T00:58:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:58:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 116082 2024-09-18T00:58:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:58:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 116084 2024-09-18T00:58:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T00:58:40.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 116085 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary1 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+replaying 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:58:40.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:40.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:58:40.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:58:40.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:58:40.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:58:41.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+replaying '' '' 2024-09-18T00:58:41.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:41.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:41.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:58:41.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:41.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:58:41.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:58:41.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:58:41.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:58:41.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T00:58:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T00:58:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9a782482-1483-42e7-a43c-95bec4be0376 2024-09-18T00:58:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:58:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70418432.0,"last_snapshot_bytes":70418432,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"} 2024-09-18T00:58:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:58:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:39 2024-09-18T00:58:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:58:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:58:41.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:58:41.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:58:41.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:39' 2024-09-18T00:58:41.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:58:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:58:41.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:58:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70418432.0,"last_snapshot_bytes":70418432,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"} 2024-09-18T00:58:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:58:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:58:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:58:41.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:58:41.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:58:41.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:58:41.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:58:41.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:58:41.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:58:41.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+replaying '2024-09-18 00:58:39' 2024-09-18T00:58:41.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:41.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:41.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:58:41.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:41.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:58:39' 2024-09-18T00:58:41.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:58:41.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:58:41.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:58:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary19a782482-1483-42e7-a43c-95bec4be0376up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70418432.0,"last_snapshot_bytes":70418432,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:39image-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":158498816.0,"last_snapshot_bytes":158498816,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726621007,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:39image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72409088.0,"last_snapshot_bytes":72409088,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:39image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72228864.0,"last_snapshot_bytes":72228864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:39image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62177280.0,"last_snapshot_bytes":62177280,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:39image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60395520.0,"last_snapshot_bytes":60395520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:39image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63729664.0,"last_snapshot_bytes":63729664,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:39image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70266880.0,"last_snapshot_bytes":70266880,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621002,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:39image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70180864.0,"last_snapshot_bytes":70180864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621010,"remote_snapshot_timestamp":1726621010,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:39image-primary993ec4895-421e-451c-941b-0c6809cf872eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62382080.0,"last_snapshot_bytes":62382080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:39' 2024-09-18T00:58:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:58:41.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T00:58:42.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:58:42.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T00:58:42.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:58:42.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:58:42.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:58:42.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:58:42.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:42.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:42.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary1 2024-09-18T00:58:42.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:58:42.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:42.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:58:42.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary1 2024-09-18T00:58:42.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary1 2024-09-18T00:58:44.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:58:44.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary1 2024-09-18T00:58:44.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:58:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:58:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:58:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary1 2024-09-18T00:58:44.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:44.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary1 2024-09-18T00:58:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:58:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:58:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary2 2024-09-18T00:58:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:58:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:58:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+replaying 2024-09-18T00:58:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:44.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:58:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:58:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:58:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:58:44.442 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:58:45.188 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:45.181+0000 7f849c00d640 -1 reset not still connected to 0x5629a85e6f70 2024-09-18T00:58:45.189 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:45.185+0000 7f849c00d640 -1 reset not still connected to 0x5629a85e7040 2024-09-18T00:58:45.194 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:45.189+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:58:45.194 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:45.189+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:58:45.194 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:45.193+0000 7f849c00d640 -1 reset not still connected to 0x5629ea430340 2024-09-18T00:58:45.194 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:45.193+0000 7f849c00d640 -1 reset not still connected to 0x5629ea430410 2024-09-18T00:58:45.196 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:45.193+0000 7f849c00d640 -1 reset not still connected to 0x5629ec6ba270 2024-09-18T00:58:45.197 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:45.193+0000 7f849c00d640 -1 reset not still connected to 0x5629f3d521a0 2024-09-18T00:58:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+replaying '' '' 2024-09-18T00:58:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:58:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:58:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:58:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:58:45.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:58:45.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T00:58:45.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T00:58:45.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a96adece-8cff-4940-898a-d2a762b46aea 2024-09-18T00:58:45.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:58:45.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72409088.0,"last_snapshot_bytes":72409088,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"} 2024-09-18T00:58:45.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:58:45.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:39 2024-09-18T00:58:45.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:58:45.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:58:45.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:58:45.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:58:45.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:44' 2024-09-18T00:58:45.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:58:45.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:58:45.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:58:45.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72409088.0,"last_snapshot_bytes":72409088,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"} 2024-09-18T00:58:45.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:58:45.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:58:45.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:58:45.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:58:45.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:58:45.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:58:45.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:58:45.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:58:45.548 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:58:45.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+replaying '2024-09-18 00:58:39' 2024-09-18T00:58:45.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:45.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:45.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:58:45.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:45.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:58:39' 2024-09-18T00:58:45.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:58:45.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:58:45.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:58:45.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":158498816.0,"last_snapshot_bytes":158498816,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726621007,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:44image-primary2a96adece-8cff-4940-898a-d2a762b46aeaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72409088.0,"last_snapshot_bytes":72409088,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:44image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72228864.0,"last_snapshot_bytes":72228864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:44image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62177280.0,"last_snapshot_bytes":62177280,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:44image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60395520.0,"last_snapshot_bytes":60395520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:44image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63729664.0,"last_snapshot_bytes":63729664,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:44image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70266880.0,"last_snapshot_bytes":70266880,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621002,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:44image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70180864.0,"last_snapshot_bytes":70180864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621010,"remote_snapshot_timestamp":1726621010,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:44image-primary993ec4895-421e-451c-941b-0c6809cf872eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62382080.0,"last_snapshot_bytes":62382080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:44' 2024-09-18T00:58:45.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:58:45.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T00:58:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:58:45.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T00:58:45.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:58:45.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:58:45.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:58:45.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:58:45.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:45.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:45.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary2 2024-09-18T00:58:45.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:58:45.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:45.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:58:45.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary2 2024-09-18T00:58:45.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary2 2024-09-18T00:58:48.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:58:48.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary2 2024-09-18T00:58:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:58:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:58:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:58:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary2 2024-09-18T00:58:48.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:48.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary2 2024-09-18T00:58:48.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:58:48.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:58:48.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary3 2024-09-18T00:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+replaying 2024-09-18T00:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:58:48.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:58:48.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:58:49.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+replaying '' '' 2024-09-18T00:58:49.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:49.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:49.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:58:49.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:58:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:58:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:58:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:58:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T00:58:49.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T00:58:49.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 238293d6-165c-443a-acb3-b579f41e40a2 2024-09-18T00:58:49.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:58:49.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72228864.0,"last_snapshot_bytes":72228864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"} 2024-09-18T00:58:49.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:58:49.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:39 2024-09-18T00:58:49.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:58:49.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:58:49.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:58:49.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:58:49.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:49' 2024-09-18T00:58:49.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:58:49.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:58:49.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:58:49.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72228864.0,"last_snapshot_bytes":72228864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"} 2024-09-18T00:58:49.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:58:49.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:58:49.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:58:49.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:58:49.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:58:49.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:58:49.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:58:49.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:58:49.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:58:49.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+replaying '2024-09-18 00:58:39' 2024-09-18T00:58:49.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:49.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:49.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:58:49.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:49.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:58:39' 2024-09-18T00:58:49.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:58:49.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:58:49.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:58:49.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK842544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":158498816.0,"last_snapshot_bytes":158498816,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726621007,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary3238293d6-165c-443a-acb3-b579f41e40a2up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72228864.0,"last_snapshot_bytes":72228864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62177280.0,"last_snapshot_bytes":62177280,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60395520.0,"last_snapshot_bytes":60395520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63729664.0,"last_snapshot_bytes":63729664,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70266880.0,"last_snapshot_bytes":70266880,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621002,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70180864.0,"last_snapshot_bytes":70180864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621010,"remote_snapshot_timestamp":1726621010,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary993ec4895-421e-451c-941b-0c6809cf872eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62382080.0,"last_snapshot_bytes":62382080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49' 2024-09-18T00:58:49.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:58:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T00:58:49.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:58:49.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T00:58:49.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:58:49.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:58:49.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:58:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:58:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary3 2024-09-18T00:58:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:58:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:58:49.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary3 2024-09-18T00:58:49.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary3 2024-09-18T00:58:51.649 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:51.645+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:58:51.651 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:51.649+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:58:51.651 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:51.649+0000 7f849c00d640 -1 reset not still connected to 0x5629ea430340 2024-09-18T00:58:51.651 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:51.649+0000 7f849c00d640 -1 reset not still connected to 0x5629ea430410 2024-09-18T00:58:51.651 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:51.649+0000 7f849c00d640 -1 reset not still connected to 0x5629f3d521a0 2024-09-18T00:58:51.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:58:51.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary3 2024-09-18T00:58:51.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:51.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:51.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:58:51.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:58:51.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:58:51.937 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary3 2024-09-18T00:58:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:51.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary3 2024-09-18T00:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary4 2024-09-18T00:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+replaying 2024-09-18T00:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:58:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:58:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:58:51.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:58:52.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+replaying '' '' 2024-09-18T00:58:52.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:52.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:58:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:58:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:58:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:58:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:58:52.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T00:58:53.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T00:58:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: cd38d20d-4581-4726-a3c5-e2f4f19b40f5 2024-09-18T00:58:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:58:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62177280.0,"last_snapshot_bytes":62177280,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"} 2024-09-18T00:58:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:58:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:39 2024-09-18T00:58:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:58:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:58:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:58:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:58:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:49' 2024-09-18T00:58:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:58:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:58:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:58:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62177280.0,"last_snapshot_bytes":62177280,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"} 2024-09-18T00:58:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:58:53.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:58:53.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:58:53.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:58:53.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:58:53.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:58:53.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:58:53.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:58:53.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:58:53.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+replaying '2024-09-18 00:58:39' 2024-09-18T00:58:53.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:53.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:53.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:58:53.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:53.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:58:39' 2024-09-18T00:58:53.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:58:53.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:58:53.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:58:53.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":158498816.0,"last_snapshot_bytes":158498816,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726621007,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary4cd38d20d-4581-4726-a3c5-e2f4f19b40f5up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62177280.0,"last_snapshot_bytes":62177280,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60395520.0,"last_snapshot_bytes":60395520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63729664.0,"last_snapshot_bytes":63729664,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70266880.0,"last_snapshot_bytes":70266880,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621002,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70180864.0,"last_snapshot_bytes":70180864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621010,"remote_snapshot_timestamp":1726621010,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49image-primary993ec4895-421e-451c-941b-0c6809cf872eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62382080.0,"last_snapshot_bytes":62382080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:49' 2024-09-18T00:58:53.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:58:53.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T00:58:53.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:58:53.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T00:58:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:58:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:58:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:58:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:58:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary4 2024-09-18T00:58:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:58:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:58:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary4 2024-09-18T00:58:53.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary4 2024-09-18T00:58:54.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary4 2024-09-18T00:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary4 2024-09-18T00:58:55.016 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:58:55.013+0000 7fac32fac640 -1 reset not still connected to 0x55a839605ba0 2024-09-18T00:58:55.016 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:58:55.013+0000 7fac32fac640 -1 reset not still connected to 0x55a83a5192b0 2024-09-18T00:58:55.016 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:58:55.013+0000 7fac32fac640 -1 reset not still connected to 0x55a83a519e10 2024-09-18T00:58:55.016 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:58:55.013+0000 7fac32fac640 -1 reset not still connected to 0x55a83cf1fa00 2024-09-18T00:58:55.016 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:58:55.013+0000 7fac32fac640 -1 reset not still connected to 0x55a84005e8f0 2024-09-18T00:58:55.020 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:58:55.017+0000 7fac32fac640 -1 reset not still connected to 0x55a84b324680 2024-09-18T00:58:55.020 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:58:55.017+0000 7fac32fac640 -1 reset not still connected to 0x55a87cb171e0 2024-09-18T00:58:55.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:55.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary4 2024-09-18T00:58:55.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:58:55.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:58:55.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary5 2024-09-18T00:58:55.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:55.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:55.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:58:55.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:58:55.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+replaying 2024-09-18T00:58:55.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:55.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:55.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:58:55.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:55.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:58:55.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:58:55.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:58:55.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:58:56.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+replaying '' '' 2024-09-18T00:58:56.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:58:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:58:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:58:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:58:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:58:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T00:58:56.037 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:56.029+0000 7f849c00d640 -1 reset not still connected to 0x5629a75ffe10 2024-09-18T00:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T00:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8b737e05-c466-4ab1-8f79-d9dc4c934fea 2024-09-18T00:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60395520.0,"last_snapshot_bytes":60395520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"} 2024-09-18T00:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:39 2024-09-18T00:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:58:56.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:54' 2024-09-18T00:58:56.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:58:56.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:58:56.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:58:56.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60395520.0,"last_snapshot_bytes":60395520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"} 2024-09-18T00:58:56.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:58:56.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:58:56.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:58:56.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:58:56.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:58:56.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:58:56.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:58:56.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:58:56.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:58:56.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+replaying '2024-09-18 00:58:39' 2024-09-18T00:58:56.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:56.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:56.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:58:56.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:58:39' 2024-09-18T00:58:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:58:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:58:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:58:56.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK642544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":158498816.0,"last_snapshot_bytes":158498816,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726621007,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:54image-primary58b737e05-c466-4ab1-8f79-d9dc4c934feaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":60395520.0,"last_snapshot_bytes":60395520,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621011,"remote_snapshot_timestamp":1726621011,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:54image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63729664.0,"last_snapshot_bytes":63729664,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:54image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70266880.0,"last_snapshot_bytes":70266880,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621002,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:54image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70180864.0,"last_snapshot_bytes":70180864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621010,"remote_snapshot_timestamp":1726621010,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:54image-primary993ec4895-421e-451c-941b-0c6809cf872eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62382080.0,"last_snapshot_bytes":62382080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:54' 2024-09-18T00:58:56.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:58:56.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T00:58:56.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:58:56.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T00:58:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:58:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:58:56.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:58:56.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:58:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary5 2024-09-18T00:58:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:58:56.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:56.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:58:56.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary5 2024-09-18T00:58:56.292 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:56.289+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:58:56.293 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:56.289+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:58:56.293 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:56.289+0000 7f849c00d640 -1 reset not still connected to 0x5629f3d521a0 2024-09-18T00:58:56.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary5 2024-09-18T00:58:57.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:58:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary5 2024-09-18T00:58:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:58:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:58:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:58:57.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary5 2024-09-18T00:58:58.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:58.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary5 2024-09-18T00:58:58.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:58:58.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:58:58.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary6 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+replaying 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:58:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:58:59.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+replaying '' '' 2024-09-18T00:58:59.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:59.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:59.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:58:59.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:59.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:58:59.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:58:59.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:58:59.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:58:59.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T00:58:59.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T00:58:59.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 77b516d0-a26f-44c3-b75d-8945de8e9052 2024-09-18T00:58:59.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:58:59.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63729664.0,"last_snapshot_bytes":63729664,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"} 2024-09-18T00:58:59.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:58:59.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:39 2024-09-18T00:58:59.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:58:59.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:58:59.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:58:59.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:58:59.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:54' 2024-09-18T00:58:59.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:58:59.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:58:59.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:58:59.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63729664.0,"last_snapshot_bytes":63729664,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"} 2024-09-18T00:58:59.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:58:59.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:58:59.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:58:59.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:58:59.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:58:59.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:58:59.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:58:59.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:58:59.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:58:59.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+replaying '2024-09-18 00:58:39' 2024-09-18T00:58:59.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:58:59.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:59.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:58:59.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:58:59.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:58:39' 2024-09-18T00:58:59.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:58:59.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:58:59.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:58:59.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":158498816.0,"last_snapshot_bytes":158498816,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726621007,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:54image-primary677b516d0-a26f-44c3-b75d-8945de8e9052up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":63729664.0,"last_snapshot_bytes":63729664,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:54image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70266880.0,"last_snapshot_bytes":70266880,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621002,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:54image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70180864.0,"last_snapshot_bytes":70180864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621010,"remote_snapshot_timestamp":1726621010,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:54image-primary993ec4895-421e-451c-941b-0c6809cf872eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62382080.0,"last_snapshot_bytes":62382080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:54' 2024-09-18T00:58:59.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:58:59.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T00:58:59.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:58:59.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T00:58:59.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:58:59.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:58:59.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:58:59.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:58:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:58:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary6 2024-09-18T00:58:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:58:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:58:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:58:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary6 2024-09-18T00:58:59.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary6 2024-09-18T00:58:59.774 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:59.768+0000 7f849c00d640 -1 reset not still connected to 0x5629a85e6f70 2024-09-18T00:58:59.774 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:59.768+0000 7f849c00d640 -1 reset not still connected to 0x5629a85e7040 2024-09-18T00:58:59.774 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:59.768+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:58:59.774 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:59.768+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:58:59.774 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:59.768+0000 7f849c00d640 -1 reset not still connected to 0x5629c7bf1860 2024-09-18T00:58:59.774 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:58:59.768+0000 7f849c00d640 -1 reset not still connected to 0x5629f3d521a0 2024-09-18T00:59:00.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:59:00.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary6 2024-09-18T00:59:00.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:00.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:00.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:00.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:00.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:59:00.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary6 2024-09-18T00:59:00.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:00.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary6 2024-09-18T00:59:00.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:00.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:59:00.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary7 2024-09-18T00:59:00.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:00.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:00.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:00.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:59:00.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+replaying 2024-09-18T00:59:00.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:00.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:00.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:00.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:00.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:59:00.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:59:00.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:00.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:59:01.471 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:59:01.468+0000 7fac32fac640 -1 reset not still connected to 0x55a83cf1fa00 2024-09-18T00:59:01.472 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T00:59:01.468+0000 7fac32fac640 -1 reset not still connected to 0x55a84b324680 2024-09-18T00:59:01.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+replaying '' '' 2024-09-18T00:59:01.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:01.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:01.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:01.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:01.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:59:01.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:59:01.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:59:01.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:59:01.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T00:59:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T00:59:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 682d0054-0ced-419a-bc1c-f9e86132b9fc 2024-09-18T00:59:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:59:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70266880.0,"last_snapshot_bytes":70266880,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621002,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"} 2024-09-18T00:59:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:59:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:39 2024-09-18T00:59:02.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:59:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:59:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:59:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:59:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:59' 2024-09-18T00:59:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:59:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:59:02.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:59:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70266880.0,"last_snapshot_bytes":70266880,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621002,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"} 2024-09-18T00:59:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:59:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:59:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:59:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:59:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:59:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:59:02.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:59:02.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:59:02.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:59:02.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+replaying '2024-09-18 00:58:39' 2024-09-18T00:59:02.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:02.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:02.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:02.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:02.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:58:39' 2024-09-18T00:59:02.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:59:02.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:59:02.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":158498816.0,"last_snapshot_bytes":158498816,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726621007,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:59image-primary7682d0054-0ced-419a-bc1c-f9e86132b9fcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70266880.0,"last_snapshot_bytes":70266880,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621002,"remote_snapshot_timestamp":1726621002,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:59image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70180864.0,"last_snapshot_bytes":70180864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621010,"remote_snapshot_timestamp":1726621010,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:59image-primary993ec4895-421e-451c-941b-0c6809cf872eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62382080.0,"last_snapshot_bytes":62382080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:58:59' 2024-09-18T00:59:02.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:59:02.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T00:59:02.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:59:02.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T00:59:02.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:59:02.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:59:02.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:59:02.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:59:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary7 2024-09-18T00:59:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:59:02.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:02.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:02.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary7 2024-09-18T00:59:02.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary7 2024-09-18T00:59:04.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:59:04.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary7 2024-09-18T00:59:04.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:04.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:04.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:04.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:04.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:59:04.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary7 2024-09-18T00:59:04.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:04.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary7 2024-09-18T00:59:04.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:04.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:59:04.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary8 2024-09-18T00:59:04.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:04.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:04.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:04.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:59:04.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+replaying 2024-09-18T00:59:04.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:04.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:04.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:04.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:04.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:59:04.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:59:04.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:04.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:59:04.297 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:59:04.292+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6734b1ee0 2024-09-18T00:59:05.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+replaying '' '' 2024-09-18T00:59:05.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:05.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:05.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:05.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:05.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:59:05.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:59:05.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:59:05.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:59:05.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T00:59:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T00:59:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: c183a554-660d-4696-b42f-a3c4a75ef26e 2024-09-18T00:59:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:59:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70180864.0,"last_snapshot_bytes":70180864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621010,"remote_snapshot_timestamp":1726621010,"replay_state":"idle"} 2024-09-18T00:59:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:59:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:39 2024-09-18T00:59:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:59:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:59:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:59:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:59:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:59:04' 2024-09-18T00:59:05.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:59:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:59:05.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:59:05.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70180864.0,"last_snapshot_bytes":70180864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621010,"remote_snapshot_timestamp":1726621010,"replay_state":"idle"} 2024-09-18T00:59:05.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:59:05.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:59:05.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:59:05.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:59:05.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:59:05.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:59:05.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:59:05.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:59:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:59:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+replaying '2024-09-18 00:58:39' 2024-09-18T00:59:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:58:39' 2024-09-18T00:59:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:59:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:59:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:59:05.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK342544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":158498816.0,"last_snapshot_bytes":158498816,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726621007,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:59:04image-primary8c183a554-660d-4696-b42f-a3c4a75ef26eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70180864.0,"last_snapshot_bytes":70180864,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621010,"remote_snapshot_timestamp":1726621010,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:59:04image-primary993ec4895-421e-451c-941b-0c6809cf872eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62382080.0,"last_snapshot_bytes":62382080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:59:04' 2024-09-18T00:59:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:59:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T00:59:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:59:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T00:59:05.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:59:05.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:59:05.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:59:05.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:59:05.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:05.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:05.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary8 2024-09-18T00:59:05.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:59:05.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:05.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:05.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary8 2024-09-18T00:59:05.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary8 2024-09-18T00:59:06.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:59:06.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary8 2024-09-18T00:59:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:59:06.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary8 2024-09-18T00:59:07.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:07.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary8 2024-09-18T00:59:07.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary9 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+replaying 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:07.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:59:08.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+replaying '' '' 2024-09-18T00:59:08.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:08.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:08.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:08.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:59:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:59:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:59:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:59:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T00:59:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T00:59:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 93ec4895-421e-451c-941b-0c6809cf872e 2024-09-18T00:59:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:59:08.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62382080.0,"last_snapshot_bytes":62382080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"} 2024-09-18T00:59:08.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:59:08.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:58:39 2024-09-18T00:59:08.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:59:08.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:59:08.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:59:08.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:59:08.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:59:04' 2024-09-18T00:59:08.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:59:08.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:59:08.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:59:08.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62382080.0,"last_snapshot_bytes":62382080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"} 2024-09-18T00:59:08.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:59:08.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:59:08.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:59:08.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:59:08.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:59:08.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:59:08.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:59:08.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:59:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:59:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+replaying '2024-09-18 00:58:39' 2024-09-18T00:59:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:58:39' 2024-09-18T00:59:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:59:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:59:08.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:59:08.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":158498816.0,"last_snapshot_bytes":158498816,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726621007,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:59:04image-primary993ec4895-421e-451c-941b-0c6809cf872eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62382080.0,"last_snapshot_bytes":62382080,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621006,"remote_snapshot_timestamp":1726621006,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:58:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:59:04' 2024-09-18T00:59:08.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:59:08.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T00:59:08.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:58:39' 2024-09-18T00:59:08.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T00:59:08.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:59:08.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:59:08.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:59:08.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:59:08.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:08.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:08.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary9 2024-09-18T00:59:08.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:59:08.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:08.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:08.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary9 2024-09-18T00:59:08.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary9 2024-09-18T00:59:10.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:59:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary9 2024-09-18T00:59:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:59:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary9 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary9 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary10 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+replaying 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:10.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:10.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:10.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:59:10.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:59:10.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:10.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T00:59:11.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+replaying '' '' 2024-09-18T00:59:11.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:11.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:11.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:11.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:11.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T00:59:11.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T00:59:11.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:59:11.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:59:11.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T00:59:11.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T00:59:11.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 6f07d2c4-2b18-4dec-be82-bb3768331b71 2024-09-18T00:59:11.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T00:59:11.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":158498816.0,"last_snapshot_bytes":158498816,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726621007,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"} 2024-09-18T00:59:11.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T00:59:11.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:59:09 2024-09-18T00:59:11.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T00:59:11.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T00:59:11.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T00:59:11.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T00:59:11.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 00:59:09' 2024-09-18T00:59:11.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:59:11.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T00:59:11.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T00:59:11.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":158498816.0,"last_snapshot_bytes":158498816,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726621007,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"} 2024-09-18T00:59:11.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T00:59:11.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:59:11.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T00:59:11.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:59:11.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T00:59:11.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T00:59:11.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T00:59:11.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T00:59:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:59:09' 2024-09-18T00:59:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+replaying '2024-09-18 00:59:09' 2024-09-18T00:59:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 00:59:09' 2024-09-18T00:59:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T00:59:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T00:59:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T00:59:11.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary106f07d2c4-2b18-4dec-be82-bb3768331b71up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":158498816.0,"last_snapshot_bytes":158498816,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1726621007,"remote_snapshot_timestamp":1726621007,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 00:59:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 00:59:09' 2024-09-18T00:59:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T00:59:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T00:59:11.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 00:59:09' 2024-09-18T00:59:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T00:59:11.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T00:59:11.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T00:59:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T00:59:11.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T00:59:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary10 2024-09-18T00:59:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T00:59:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:11.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary10 2024-09-18T00:59:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary10 2024-09-18T00:59:13.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T00:59:13.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary10 2024-09-18T00:59:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T00:59:13.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary10 2024-09-18T00:59:13.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:13.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary10 2024-09-18T00:59:13.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..10} 2024-09-18T00:59:13.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS=() 2024-09-18T00:59:13.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS=() 2024-09-18T00:59:13.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS=() 2024-09-18T00:59:13.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS=() 2024-09-18T00:59:13.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RET=0 2024-09-18T00:59:13.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:13.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T00:59:13.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary1 2024-09-18T00:59:13.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary1 snapshot 10G 2024-09-18T00:59:13.276 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary1 10G 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:13.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:13.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:13.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:59:13.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:59:13.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:59:13.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:13.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary1 2024-09-18T00:59:13.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:59:13.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:13.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:59:13.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:13.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary1 snapshot 2024-09-18T00:59:13.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:13.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:13.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:13.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:13.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary1 snapshot 2024-09-18T00:59:14.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:59:14.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary1 2024-09-18T00:59:14.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary1': 2024-09-18T00:59:14.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:59:14.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:59:14.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:59:14.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 34f0fb3dc6ce 2024-09-18T00:59:14.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.34f0fb3dc6ce 2024-09-18T00:59:14.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:59:14.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:59:14.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:59:14.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:59:14.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:59:13 2024 2024-09-18T00:59:14.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:59:13 2024 2024-09-18T00:59:14.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:59:13 2024 2024-09-18T00:59:14.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:59:14.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:59:14.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: a61c5b8e-bc8f-4d8e-b48f-991b9b77e317 2024-09-18T00:59:14.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:59:14.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:59:14.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T00:59:14.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd0 2024-09-18T00:59:14.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:59:14.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd0 2024-09-18T00:59:14.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:59:14.385 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:59:14.380+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4750 2024-09-18T00:59:14.392 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:59:14.384+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6734b1ee0 2024-09-18T00:59:14.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:59:14.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:59:14.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 6a6dec5d-ae44-4620-8b22-35a501b0838f 2024-09-18T00:59:14.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:59:14.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:59:14.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:14.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:59:14.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:59:14.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:59:14.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:59:14.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:14.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary1 2024-09-18T00:59:15.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd0 test-primary1 2024-09-18T00:59:15.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:59:15.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary1 2024-09-18T00:59:15.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:59:15.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary1 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary1 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T00:59:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary2 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary2 snapshot 10G 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary1 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary1 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:59:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary2 10G 2024-09-18T00:59:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:15.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:59:15.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:59:15.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:59:15.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:15.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary2 2024-09-18T00:59:15.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:59:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:59:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary2 snapshot 2024-09-18T00:59:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:15.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary2 snapshot 2024-09-18T00:59:15.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2647 2024-09-18T00:59:15.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:16.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:59:16.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary1/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary1' 2024-09-18T00:59:16.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:59:16.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary2 2024-09-18T00:59:16.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary2': 2024-09-18T00:59:16.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:59:16.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:59:16.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:59:16.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 34fe7d916f44 2024-09-18T00:59:16.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.34fe7d916f44 2024-09-18T00:59:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:59:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:59:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:59:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:59:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:59:15 2024 2024-09-18T00:59:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:59:15 2024 2024-09-18T00:59:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:59:15 2024 2024-09-18T00:59:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:59:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:59:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 640308f0-3a45-4793-82ba-ab0e29660b48 2024-09-18T00:59:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:59:16.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:59:16.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T00:59:16.682 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd1 2024-09-18T00:59:16.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:59:16.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd1 2024-09-18T00:59:16.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:59:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:59:16.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:59:16.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 49b5f5e2-f49c-465d-b178-dc2a6649c3d6 2024-09-18T00:59:16.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:59:16.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:59:16.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:16.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:59:16.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:59:17.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:59:17.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:59:17.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:17.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary2 2024-09-18T00:59:17.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd1 test-primary2 2024-09-18T00:59:17.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:59:17.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary2 2024-09-18T00:59:17.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:59:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary2 2024-09-18T00:59:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:59:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:59:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T00:59:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary3 2024-09-18T00:59:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary2 2024-09-18T00:59:17.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary3 snapshot 10G 2024-09-18T00:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary3 10G 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary2 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary2 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:59:17.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:17.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary3 2024-09-18T00:59:17.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:59:17.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:17.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:59:17.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:17.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary3 snapshot 2024-09-18T00:59:17.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:17.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:17.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:17.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:17.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary3 snapshot 2024-09-18T00:59:17.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:59:17.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary2/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary2' 2024-09-18T00:59:17.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2649 2024-09-18T00:59:17.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:59:18.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary3 2024-09-18T00:59:18.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary3': 2024-09-18T00:59:18.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:59:18.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:59:18.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:59:18.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 350ee3de6f2f 2024-09-18T00:59:18.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.350ee3de6f2f 2024-09-18T00:59:18.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:59:18.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:59:18.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:59:18.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:59:18.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:59:17 2024 2024-09-18T00:59:18.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:59:17 2024 2024-09-18T00:59:18.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:59:17 2024 2024-09-18T00:59:18.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:59:18.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:59:18.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 4a901bdb-c18f-43d9-a0f0-efe4d94ef3aa 2024-09-18T00:59:18.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:59:18.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:59:18.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T00:59:18.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd2 2024-09-18T00:59:18.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:59:18.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd2 2024-09-18T00:59:18.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:59:18.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:59:18.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:59:18.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: ce2ae8b8-82a4-4341-ad08-025468020d63 2024-09-18T00:59:18.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:59:18.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:59:18.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:18.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:59:18.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:59:18.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:18.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:59:18.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:18.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:18.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:18.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:59:18.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:59:18.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:59:18.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:18.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary3 2024-09-18T00:59:18.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd2 test-primary3 2024-09-18T00:59:18.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:59:18.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary3 2024-09-18T00:59:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:59:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary3 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary4 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:59:18.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary4 snapshot 10G 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary4 10G 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:18.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:18.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:59:18.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:18.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:59:18.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:59:18.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:59:18.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:18.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary4 2024-09-18T00:59:18.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary3 2024-09-18T00:59:18.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary3 2024-09-18T00:59:18.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary3 2024-09-18T00:59:19.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:59:19.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:19.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:59:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary4 snapshot 2024-09-18T00:59:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:59:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:19.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary4 snapshot 2024-09-18T00:59:19.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:59:19.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary3/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary3' 2024-09-18T00:59:19.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2651 2024-09-18T00:59:20.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:20.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2652 2024-09-18T00:59:20.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:59:20.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:20.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary4 2024-09-18T00:59:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary4': 2024-09-18T00:59:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:59:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:59:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:59:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 351e7abffc20 2024-09-18T00:59:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.351e7abffc20 2024-09-18T00:59:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:59:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:59:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:59:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:59:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:59:19 2024 2024-09-18T00:59:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:59:19 2024 2024-09-18T00:59:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:59:19 2024 2024-09-18T00:59:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:59:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:59:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: f16bbff5-762b-414e-93aa-bcfe90e66fb3 2024-09-18T00:59:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:59:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:59:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T00:59:20.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd3 2024-09-18T00:59:20.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:59:20.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd3 2024-09-18T00:59:20.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:59:20.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:59:20.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:59:20.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 8e1b70fb-f128-43d3-8775-3ac849dbdc85 2024-09-18T00:59:20.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:59:20.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:59:20.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:20.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:59:20.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:59:20.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:20.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:59:20.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:20.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:20.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:20.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:59:20.682 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:59:20.676+0000 7f849c00d640 -1 reset not still connected to 0x5629a85e6f70 2024-09-18T00:59:20.683 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:59:20.676+0000 7f849c00d640 -1 reset not still connected to 0x5629a85e7040 2024-09-18T00:59:20.687 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:59:20.684+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T00:59:20.688 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T00:59:20.684+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T00:59:20.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:59:20.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:59:20.914 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:20.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary4 2024-09-18T00:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd3 test-primary4 2024-09-18T00:59:21.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:59:21.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary4 2024-09-18T00:59:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:59:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary4 2024-09-18T00:59:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:59:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:59:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:59:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary4 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary5 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary5 snapshot 10G 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary4 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary4 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary5 10G 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:59:21.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:59:21.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:21.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary5 2024-09-18T00:59:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:59:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:59:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary5 snapshot 2024-09-18T00:59:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:59:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary5 snapshot 2024-09-18T00:59:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:59:21.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary4/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary4' 2024-09-18T00:59:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2654 2024-09-18T00:59:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:22.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2655 2024-09-18T00:59:22.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:22.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:59:22.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary5 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary5': 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 353030d088c6 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.353030d088c6 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:59:21 2024 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:59:21 2024 2024-09-18T00:59:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:59:21 2024 2024-09-18T00:59:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:59:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:59:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 5984c013-9b62-4bf1-bc42-f8038f39f85c 2024-09-18T00:59:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:59:22.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:59:22.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T00:59:22.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd4 2024-09-18T00:59:22.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:59:22.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd4 2024-09-18T00:59:22.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:59:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:59:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:59:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 7dfb57c0-d4bc-4a99-b773-c80bf8e6d823 2024-09-18T00:59:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:59:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:59:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:59:22.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:59:22.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:59:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:59:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:22.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary5 2024-09-18T00:59:22.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd4 test-primary5 2024-09-18T00:59:22.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:59:22.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary5 2024-09-18T00:59:23.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:23.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:59:23.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:59:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:59:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary5 2024-09-18T00:59:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:59:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary5 2024-09-18T00:59:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T00:59:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary6 2024-09-18T00:59:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary6 snapshot 10G 2024-09-18T00:59:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary5 2024-09-18T00:59:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary5 2024-09-18T00:59:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:59:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:23.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary6 10G 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:59:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:59:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:59:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:59:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:23.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary6 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:59:23.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:59:23.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:23.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:59:23.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:23.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary6 snapshot 2024-09-18T00:59:23.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:23.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:23.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:23.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:23.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary6 snapshot 2024-09-18T00:59:23.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:59:23.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary5/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary5' 2024-09-18T00:59:24.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2657 2024-09-18T00:59:24.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:24.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:59:24.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary6 2024-09-18T00:59:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary6': 2024-09-18T00:59:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:59:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:59:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:59:24.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 353f6fd5a853 2024-09-18T00:59:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.353f6fd5a853 2024-09-18T00:59:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:59:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:59:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:59:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:59:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:59:23 2024 2024-09-18T00:59:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:59:23 2024 2024-09-18T00:59:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:59:23 2024 2024-09-18T00:59:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:59:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:59:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 97f2fe38-61d0-453f-b29c-085ba52e905a 2024-09-18T00:59:24.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:59:24.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:59:24.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T00:59:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd5 2024-09-18T00:59:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:59:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd5 2024-09-18T00:59:24.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:59:24.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:59:24.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:59:24.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: fdde1509-9ed0-4cbe-a595-3b250718b5c0 2024-09-18T00:59:24.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:59:24.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:59:24.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:24.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:59:24.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:59:24.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:59:25.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:25.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:59:25.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:25.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:25.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:25.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:59:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:59:25.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:25.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary6 2024-09-18T00:59:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd5 test-primary6 2024-09-18T00:59:25.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:25.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:59:25.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:25.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:25.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:59:25.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:59:25.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2658 2024-09-18T00:59:25.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2659 2024-09-18T00:59:25.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:25.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:59:25.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary6 2024-09-18T00:59:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:59:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary6 2024-09-18T00:59:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:59:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary6 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary7 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary7 snapshot 10G 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary6 2024-09-18T00:59:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary6 2024-09-18T00:59:25.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:25.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:25.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:25.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:25.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:59:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary7 10G 2024-09-18T00:59:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary7 2024-09-18T00:59:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:59:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:59:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary7 snapshot 2024-09-18T00:59:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:25.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary7 snapshot 2024-09-18T00:59:25.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:59:25.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary6/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary6' 2024-09-18T00:59:26.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2663 2024-09-18T00:59:26.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:26.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:59:26.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary7 2024-09-18T00:59:26.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary7': 2024-09-18T00:59:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:59:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:59:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:59:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 355ab2a2e5bc 2024-09-18T00:59:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.355ab2a2e5bc 2024-09-18T00:59:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:59:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:59:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:59:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:59:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:59:25 2024 2024-09-18T00:59:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:59:25 2024 2024-09-18T00:59:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:59:25 2024 2024-09-18T00:59:26.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:59:26.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:59:26.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 225f2829-01d3-45e2-a1d6-0f790148e4c3 2024-09-18T00:59:26.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:59:26.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:59:26.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T00:59:26.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd6 2024-09-18T00:59:26.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:59:26.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd6 2024-09-18T00:59:26.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:59:26.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:59:26.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:59:26.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 56de435e-aab7-4a6a-b116-6b3faeac6128 2024-09-18T00:59:26.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:59:26.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:59:26.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:26.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:59:26.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:59:26.792 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:59:26.788+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6734b1ee0 2024-09-18T00:59:26.792 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:59:26.788+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735afee0 2024-09-18T00:59:26.792 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T00:59:26.788+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d679f90000 2024-09-18T00:59:27.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): Snapshot ID: 2662 2024-09-18T00:59:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:27.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:27.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:59:27.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:27.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:27.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:59:27.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:59:27.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:59:27.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 Snapshot ID: 2661 2024-09-18T00:59:27.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:27.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:59:27.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:27.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary7 2024-09-18T00:59:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd6 test-primary7 2024-09-18T00:59:27.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:59:27.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary7 2024-09-18T00:59:27.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:59:27.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary7 2024-09-18T00:59:27.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:27.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:59:27.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary7 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary8 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary8 snapshot 10G 2024-09-18T00:59:27.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary7 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary7 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary8 10G 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:27.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:27.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:27.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:27.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:27.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:27.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:59:27.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:59:27.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:59:27.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:27.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary8 2024-09-18T00:59:27.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:59:27.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary7/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary7' 2024-09-18T00:59:27.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:59:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:59:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary8 snapshot 2024-09-18T00:59:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:27.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary8 snapshot 2024-09-18T00:59:28.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2666 2024-09-18T00:59:28.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:28.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:59:28.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:28.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:59:28.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:28.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:28.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:28.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:59:28.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary8 2024-09-18T00:59:28.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:28.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:59:28.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:28.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:28.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:28.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:59:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary8': 2024-09-18T00:59:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:59:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:59:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:59:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 356b11d13137 2024-09-18T00:59:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.356b11d13137 2024-09-18T00:59:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:59:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:59:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:59:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:59:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:59:27 2024 2024-09-18T00:59:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:59:27 2024 2024-09-18T00:59:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:59:27 2024 2024-09-18T00:59:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:59:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:59:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: f1cbc639-19c2-4066-bbe2-5861c8b3f55a 2024-09-18T00:59:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:59:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:59:28.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T00:59:28.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd7 2024-09-18T00:59:28.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:59:28.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd7 2024-09-18T00:59:28.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:59:28.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:59:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:59:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 60516824-d838-4976-9afa-ff6c205eaee9 2024-09-18T00:59:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:59:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:59:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:59:28.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:59:28.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T00:59:28.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:59:28.961 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:28.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary8 2024-09-18T00:59:28.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd7 test-primary8 2024-09-18T00:59:29.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2665 2024-09-18T00:59:29.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:29.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:29.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:59:29.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:29.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:29.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:29.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:59:29.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:59:29.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary8 2024-09-18T00:59:29.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:59:29.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary8 2024-09-18T00:59:29.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:29.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary9 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary9 snapshot 10G 2024-09-18T00:59:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary8 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary9 10G 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary8 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary8 2024-09-18T00:59:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:59:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:59:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:59:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:29.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary9 2024-09-18T00:59:29.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:59:29.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary8/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary8' 2024-09-18T00:59:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:59:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:59:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:29.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary9 snapshot 2024-09-18T00:59:29.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:29.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:29.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:29.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:29.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary9 snapshot 2024-09-18T00:59:30.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2669 2024-09-18T00:59:30.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:30.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:59:30.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:30.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:30.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:59:30.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:59:30.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2668 2024-09-18T00:59:30.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:30.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2670 2024-09-18T00:59:30.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:30.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:59:30.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary9 2024-09-18T00:59:30.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary9': 2024-09-18T00:59:30.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:59:30.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:59:30.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:59:30.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 3582e590e953 2024-09-18T00:59:30.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.3582e590e953 2024-09-18T00:59:30.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:59:30.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:59:30.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:59:30.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:59:30.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:59:29 2024 2024-09-18T00:59:30.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:59:29 2024 2024-09-18T00:59:30.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:59:29 2024 2024-09-18T00:59:30.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:59:30.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:59:30.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 36b71294-3a4d-45ce-bfdd-d89c2dfd7337 2024-09-18T00:59:30.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:59:30.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:59:30.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T00:59:30.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:30.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:59:30.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:30.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:30.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:30.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:59:30.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd8 2024-09-18T00:59:30.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:59:30.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd8 2024-09-18T00:59:30.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:59:30.704 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-18T00:59:30.695+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0dabb790 2024-09-18T00:59:30.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:59:30.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:59:30.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 45dcea66-ac35-468c-a21b-4fc0f25dbfd4 2024-09-18T00:59:30.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:59:30.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:59:30.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:30.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:59:30.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:59:31.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): Snapshot ID: 2671 2024-09-18T00:59:31.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:31.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:31.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:59:31.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:31.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:31.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:31.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:59:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:59:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:59:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary9 2024-09-18T00:59:31.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd8 test-primary9 2024-09-18T00:59:31.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:59:31.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary9 2024-09-18T00:59:31.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:59:31.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary9 2024-09-18T00:59:31.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:31.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:31.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:31.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:31.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:59:31.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:31.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:31.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:31.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:59:31.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:59:31.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T00:59:31.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T00:59:31.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary10 2024-09-18T00:59:31.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary10 snapshot 10G 2024-09-18T00:59:31.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:31.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:31.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:31.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:31.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:31.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:31.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:31.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T00:59:31.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:31.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary10 10G 2024-09-18T00:59:31.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:31.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:31.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:31.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary9 2024-09-18T00:59:31.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:31.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:31.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:31.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T00:59:31.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T00:59:31.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T00:59:31.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T00:59:31.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary10 2024-09-18T00:59:31.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary9 2024-09-18T00:59:31.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary9 2024-09-18T00:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T00:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T00:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary10 snapshot 2024-09-18T00:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T00:59:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary10 snapshot 2024-09-18T00:59:31.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:59:31.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary9/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary9' 2024-09-18T00:59:32.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:32.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:59:32.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:32.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:32.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:59:32.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:59:32.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2674 2024-09-18T00:59:32.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2673 2024-09-18T00:59:32.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:32.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:32.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T00:59:32.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2676 2024-09-18T00:59:32.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:32.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary10 2024-09-18T00:59:32.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary10': 2024-09-18T00:59:32.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T00:59:32.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T00:59:32.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T00:59:32.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 359cee4bb670 2024-09-18T00:59:32.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.359cee4bb670 2024-09-18T00:59:32.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T00:59:32.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T00:59:32.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T00:59:32.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T00:59:32.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 00:59:31 2024 2024-09-18T00:59:32.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 00:59:31 2024 2024-09-18T00:59:32.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 00:59:31 2024 2024-09-18T00:59:32.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T00:59:32.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T00:59:32.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 943aeb36-cdea-4614-a073-08041a451b48 2024-09-18T00:59:32.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T00:59:32.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T00:59:32.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T00:59:32.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd9 2024-09-18T00:59:32.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T00:59:32.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd9 2024-09-18T00:59:32.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T00:59:32.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T00:59:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T00:59:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 2775f26b-6b83-49a6-9f07-4ef94b0aba36 2024-09-18T00:59:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T00:59:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T00:59:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T00:59:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T00:59:33.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): Snapshot ID: 2675 2024-09-18T00:59:33.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:33.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:33.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:59:33.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:33.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:33.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:33.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:59:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:59:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:33.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:59:33.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:33.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:59:33.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:33.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:33.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:33.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:59:33.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T00:59:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T00:59:33.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T00:59:33.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary10 2024-09-18T00:59:33.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd9 test-primary10 2024-09-18T00:59:33.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T00:59:33.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary10 2024-09-18T00:59:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T00:59:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary10 2024-09-18T00:59:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T00:59:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:59:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary10 2024-09-18T00:59:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T00:59:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 119807 2024-09-18T00:59:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:59:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary10 2024-09-18T00:59:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary10 2024-09-18T00:59:33.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T00:59:33.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary10/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary10' 2024-09-18T00:59:34.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2678 2024-09-18T00:59:34.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:34.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:59:34.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:34.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:34.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:34.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:59:34.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:34.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2682 2024-09-18T00:59:34.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:35.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2680 2024-09-18T00:59:35.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2679 2024-09-18T00:59:35.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:35.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2681 2024-09-18T00:59:35.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:35.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:35.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:59:35.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:35.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:35.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:35.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:59:35.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:35.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:35.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:59:35.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:35.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:35.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:59:35.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:59:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:59:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:59:36.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:36.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:59:36.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:36.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:36.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:36.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:59:36.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2683 2024-09-18T00:59:36.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:37.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:37.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:59:37.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:37.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:37.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:59:37.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:59:37.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2685 2024-09-18T00:59:37.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:37.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2684 2024-09-18T00:59:37.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2686 2024-09-18T00:59:37.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:37.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:37.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:37.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:59:37.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:37.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:37.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:37.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:59:37.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2687 2024-09-18T00:59:37.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:38.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:38.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:59:38.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:38.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:38.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:38.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:59:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:59:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:59:38.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:38.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:59:38.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:38.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:38.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:38.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:59:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:59:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:39.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:59:39.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2689 2024-09-18T00:59:39.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2688 2024-09-18T00:59:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:39.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:40.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2690 2024-09-18T00:59:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:59:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:59:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:59:40.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2692 2024-09-18T00:59:40.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2691 2024-09-18T00:59:40.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:59:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:59:40.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:40.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:40.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:40.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:59:40.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:40.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:40.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:40.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:59:41.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:41.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:59:41.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:41.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:41.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:41.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:59:41.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2693 2024-09-18T00:59:41.144 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:42.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2696 2024-09-18T00:59:42.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:42.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:59:42.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:42.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:42.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:42.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:59:42.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2694 2024-09-18T00:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:59:42.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:59:42.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2695 2024-09-18T00:59:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:42.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:42.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:43.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2697 2024-09-18T00:59:43.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:43.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:43.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:59:43.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:43.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:43.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:43.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:59:43.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:43.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:59:43.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:43.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:43.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:43.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:59:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:59:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:59:44.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2699 2024-09-18T00:59:44.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2698 2024-09-18T00:59:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:59:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:44.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:44.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:59:44.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:44.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:59:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:45.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:59:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2701 2024-09-18T00:59:45.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2702 2024-09-18T00:59:45.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:45.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:59:45.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:45.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:45.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:59:45.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:59:45.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:45.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:59:45.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:45.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:45.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:45.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:59:45.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2700 2024-09-18T00:59:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:45.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:45.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:59:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:59:46.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2703 2024-09-18T00:59:46.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:47.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2705 2024-09-18T00:59:47.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:47.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:59:47.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:47.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:47.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:59:47.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:59:47.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2704 2024-09-18T00:59:47.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:47.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:59:47.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:47.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:47.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:47.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:59:47.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2706 2024-09-18T00:59:47.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:47.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:47.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2707 2024-09-18T00:59:48.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:48.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:59:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:48.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:59:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:59:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:59:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:59:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:48.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:59:49.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2709 2024-09-18T00:59:49.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:49.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:49.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:59:49.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:49.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:49.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:49.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:59:49.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2708 2024-09-18T00:59:49.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:50.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2712 2024-09-18T00:59:50.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2710 2024-09-18T00:59:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:59:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:59:50.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2711 2024-09-18T00:59:50.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:59:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:59:50.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:59:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:59:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:50.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:59:50.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:51.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:51.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:59:51.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:51.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:51.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:51.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:59:51.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2713 2024-09-18T00:59:51.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:52.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:52.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:59:52.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:52.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:52.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:59:52.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:59:52.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2715 2024-09-18T00:59:52.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2716 2024-09-18T00:59:52.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:52.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:59:52.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:52.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:52.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:52.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:59:52.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:52.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2714 2024-09-18T00:59:52.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:52.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:53.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2717 2024-09-18T00:59:53.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:53.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:53.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:59:53.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:53.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:53.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:53.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:59:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:53.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:59:53.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:53.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:53.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:53.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:59:53.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:53.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:59:53.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:53.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:53.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:53.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:59:54.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2719 2024-09-18T00:59:54.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:54.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:59:54.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:54.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:54.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:54.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T00:59:54.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:54.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2718 2024-09-18T00:59:54.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:55.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2720 2024-09-18T00:59:55.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2721 2024-09-18T00:59:55.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:55.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T00:59:55.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:55.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:55.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T00:59:55.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T00:59:55.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:55.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2722 2024-09-18T00:59:55.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:55.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T00:59:55.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:55.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:55.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T00:59:55.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T00:59:55.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:55.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T00:59:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T00:59:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T00:59:55.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:56.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:56.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T00:59:56.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:56.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:56.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T00:59:56.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T00:59:56.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2723 2024-09-18T00:59:56.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:57.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:57.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T00:59:57.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:57.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:57.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T00:59:57.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T00:59:57.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2725 2024-09-18T00:59:57.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2724 2024-09-18T00:59:57.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2726 2024-09-18T00:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T00:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T00:59:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T00:59:57.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:57.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:57.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:58.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2727 2024-09-18T00:59:58.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:58.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:58.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T00:59:58.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:58.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:58.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T00:59:58.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T00:59:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T00:59:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T00:59:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T00:59:58.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:58.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T00:59:58.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:58.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:58.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T00:59:58.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T00:59:59.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2728 2024-09-18T00:59:59.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2729 2024-09-18T00:59:59.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:59.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T00:59:59.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T00:59:59.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T00:59:59.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T00:59:59.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T00:59:59.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T00:59:59.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:00:00.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2731 2024-09-18T01:00:00.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2730 2024-09-18T01:00:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2732 2024-09-18T01:00:00.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:00.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:00.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:00.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:00.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:00:00.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:00.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:00.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:00:00.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:00:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:00:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:00:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:00:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:00:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:00:00.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:00:01.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:01.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:00:01.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:01.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:01.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:00:01.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:00:01.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2733 2024-09-18T01:00:01.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:02.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2735 2024-09-18T01:00:02.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2734 2024-09-18T01:00:02.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2736 2024-09-18T01:00:02.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:02.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:00:02.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:02.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:02.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:00:02.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:00:02.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:02.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:02.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:00:02.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:02.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:02.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:00:02.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:00:02.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:03.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2737 2024-09-18T01:00:03.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:03.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:00:03.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:03.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:03.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:00:03.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:00:03.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:03.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:00:03.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:03.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:03.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:00:03.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:00:03.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:03.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:03.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:00:03.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:03.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:03.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:00:03.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:00:04.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:04.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:00:04.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:04.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:04.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:00:04.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:00:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2738 2024-09-18T01:00:04.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:04.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2739 2024-09-18T01:00:04.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:04.157 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T01:00:04.154+0000 7fac32fac640 -1 reset not still connected to 0x55a85dff7040 2024-09-18T01:00:04.157 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T01:00:04.154+0000 7fac32fac640 -1 reset not still connected to 0x55a85eb8a9c0 2024-09-18T01:00:05.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:05.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:00:05.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:05.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:05.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:00:05.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:00:05.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:00:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:00:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:00:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:00:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:00:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:00:05.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2740 2024-09-18T01:00:05.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2742 2024-09-18T01:00:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2741 2024-09-18T01:00:05.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:05.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:05.117 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:06.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:06.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:00:06.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:06.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:06.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:00:06.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:00:06.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2743 2024-09-18T01:00:06.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:07.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:07.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:00:07.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:07.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:07.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:00:07.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:00:07.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2745 2024-09-18T01:00:07.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:07.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:07.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:00:07.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:07.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:07.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:00:07.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:00:07.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2744 2024-09-18T01:00:07.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2746 2024-09-18T01:00:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:07.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:08.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2747 2024-09-18T01:00:08.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:08.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:08.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:00:08.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:08.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:08.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:00:08.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:00:08.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:08.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:00:08.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:08.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:08.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:00:08.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:00:09.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:09.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:00:09.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:09.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:09.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:00:09.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:00:09.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2749 2024-09-18T01:00:09.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2748 2024-09-18T01:00:09.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:09.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:10.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2750 2024-09-18T01:00:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:00:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:00:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:00:10.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2751 2024-09-18T01:00:10.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:10.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2752 2024-09-18T01:00:10.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:10.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:00:10.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:10.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:10.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:00:10.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:00:10.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:00:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:00:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:00:10.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:11.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:11.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:00:11.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:11.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:00:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:00:11.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2753 2024-09-18T01:00:11.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:11.774 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:00:11.770+0000 7f849c00d640 -1 reset not still connected to 0x5629c077c5b0 2024-09-18T01:00:11.774 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:00:11.770+0000 7f849c00d640 -1 reset not still connected to 0x5629d13be750 2024-09-18T01:00:11.776 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:00:11.774+0000 7f849c00d640 -1 reset not still connected to 0x5629d3685e10 2024-09-18T01:00:11.777 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:00:11.774+0000 7f849c00d640 -1 reset not still connected to 0x5629ed2bf1e0 2024-09-18T01:00:11.777 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:00:11.774+0000 7f849c00d640 -1 reset not still connected to 0x5629f3c5cc30 2024-09-18T01:00:12.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2755 2024-09-18T01:00:12.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2754 2024-09-18T01:00:12.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2756 2024-09-18T01:00:12.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:12.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:00:12.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:12.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:12.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:00:12.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:00:12.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:12.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:12.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:00:12.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:12.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:12.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:00:12.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:00:12.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:12.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:13.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2757 2024-09-18T01:00:13.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:00:13.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:00:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:00:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:00:13.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:13.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:13.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:00:13.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:00:13.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:13.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:00:13.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:13.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:13.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:00:13.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:00:14.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2758 2024-09-18T01:00:14.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:14.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:00:14.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:14.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:14.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:00:14.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:00:14.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2759 2024-09-18T01:00:14.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:15.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2760 2024-09-18T01:00:15.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:15.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2762 2024-09-18T01:00:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:00:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:00:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:00:15.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:15.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2761 2024-09-18T01:00:15.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:15.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:00:15.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:15.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:15.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:00:15.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:00:15.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:15.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:00:15.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:15.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:15.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:00:15.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:00:15.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:16.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:16.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:00:16.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:16.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:16.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:00:16.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:00:16.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2763 2024-09-18T01:00:16.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:17.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2765 2024-09-18T01:00:17.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:17.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:00:17.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:17.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:17.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:00:17.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:00:17.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2766 2024-09-18T01:00:17.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2764 2024-09-18T01:00:17.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:17.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:17.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:17.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:17.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:00:17.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:17.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:17.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:00:17.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:00:18.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2767 2024-09-18T01:00:18.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:18.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:18.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:00:18.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:18.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:18.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:00:18.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:00:18.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:18.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:00:18.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:18.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:18.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:00:18.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:00:18.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:18.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:00:18.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:18.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:18.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:00:18.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:00:19.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:19.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:00:19.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:19.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:19.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:00:19.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:00:19.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2769 2024-09-18T01:00:19.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2768 2024-09-18T01:00:19.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:19.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:20.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:20.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:00:20.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:20.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:20.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:00:20.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:00:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:00:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:00:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:00:20.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2772 2024-09-18T01:00:20.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:00:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:00:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:00:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2771 2024-09-18T01:00:20.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2770 2024-09-18T01:00:20.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:20.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:20.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:21.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:00:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:00:21.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:00:21.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2773 2024-09-18T01:00:21.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:22.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2774 2024-09-18T01:00:22.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2775 2024-09-18T01:00:22.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2776 2024-09-18T01:00:22.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:22.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:22.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:22.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:22.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:00:22.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:22.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:22.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:00:22.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:00:22.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:22.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:00:22.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:22.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:22.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:00:22.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:00:23.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2777 2024-09-18T01:00:23.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:23.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:23.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:00:23.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:00:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:00:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:00:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:00:23.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:00:23.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:23.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:00:23.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:23.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:23.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:00:23.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2778 2024-09-18T01:00:24.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2779 2024-09-18T01:00:24.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:24.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:24.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:24.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:00:24.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:24.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:24.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:00:24.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:00:25.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2781 2024-09-18T01:00:25.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:25.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:00:25.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:25.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:25.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:00:25.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:00:25.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2782 2024-09-18T01:00:25.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:25.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:00:25.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:25.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:25.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:00:25.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:00:25.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2780 2024-09-18T01:00:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:00:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:00:25.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:00:25.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:25.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:25.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:26.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:26.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:00:26.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:26.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:26.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:00:26.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:00:26.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2783 2024-09-18T01:00:26.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:00:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:27.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:27.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:00:27.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:00:27.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:27.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:00:27.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:27.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:27.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:00:27.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:00:27.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2786 2024-09-18T01:00:27.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2784 2024-09-18T01:00:27.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2785 2024-09-18T01:00:27.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:27.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:27.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:28.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2787 2024-09-18T01:00:28.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:28.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:28.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:00:28.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:28.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:28.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:00:28.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:00:28.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:28.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:00:28.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:28.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:28.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:00:28.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:00:28.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:28.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:00:28.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:28.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:28.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:00:28.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:00:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2789 2024-09-18T01:00:29.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2788 2024-09-18T01:00:29.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:00:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:00:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:00:30.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2791 2024-09-18T01:00:30.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2792 2024-09-18T01:00:30.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2790 2024-09-18T01:00:30.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:30.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:30.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:30.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:30.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:00:30.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:30.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:30.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:00:30.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:00:30.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:30.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:00:30.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:30.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:30.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:00:30.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:00:30.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:30.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:00:30.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:30.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:30.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:00:30.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:00:31.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2793 2024-09-18T01:00:31.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:31.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:00:31.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:31.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:31.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:00:31.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:00:31.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:32.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2796 2024-09-18T01:00:32.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2794 2024-09-18T01:00:32.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2795 2024-09-18T01:00:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:00:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:32.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:32.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:00:32.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:00:32.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:32.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:32.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:32.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:00:32.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:32.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:32.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:00:32.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:00:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2797 2024-09-18T01:00:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:00:33.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:33.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:33.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:00:33.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:00:33.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:33.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:00:33.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:33.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:33.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:00:33.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:00:33.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:00:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:33.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:00:33.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:00:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:00:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:00:34.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:00:34.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2799 2024-09-18T01:00:34.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2798 2024-09-18T01:00:34.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:34.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:35.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2801 2024-09-18T01:00:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:00:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:00:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:00:35.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:00:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:00:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:00:35.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2802 2024-09-18T01:00:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:00:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:00:35.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:00:35.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2800 2024-09-18T01:00:35.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:35.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:35.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:00:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:00:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:00:36.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2803 2024-09-18T01:00:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2805 2024-09-18T01:00:37.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2806 2024-09-18T01:00:37.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:37.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:00:37.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:37.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:37.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:00:37.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:00:37.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2804 2024-09-18T01:00:37.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:37.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:00:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:00:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:00:37.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:38.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2807 2024-09-18T01:00:38.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:00:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:00:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:00:38.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:38.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:00:38.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:38.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:38.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:00:38.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:00:38.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:38.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:00:38.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:38.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:38.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:00:38.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:00:39.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:39.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2808 2024-09-18T01:00:39.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:00:39.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:39.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:39.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:00:39.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:00:39.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2809 2024-09-18T01:00:39.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:00:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:00:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:00:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:00:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:40.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:00:40.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:00:40.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2812 2024-09-18T01:00:40.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2811 2024-09-18T01:00:40.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2810 2024-09-18T01:00:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:00:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:00:40.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:00:40.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:40.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:40.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:41.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:41.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:00:41.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:41.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:41.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:00:41.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:00:41.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2813 2024-09-18T01:00:41.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:42.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2816 2024-09-18T01:00:42.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:42.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:00:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:00:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:00:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:00:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:00:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:00:42.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2815 2024-09-18T01:00:42.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2814 2024-09-18T01:00:42.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:42.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:42.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:43.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2817 2024-09-18T01:00:43.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:43.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:43.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:00:43.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:43.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:43.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:00:43.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:00:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:00:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:00:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:00:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:00:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:43.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:00:43.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:00:44.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2818 2024-09-18T01:00:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2819 2024-09-18T01:00:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:00:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:44.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:44.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:00:44.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:00:44.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:44.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:45.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2822 2024-09-18T01:00:45.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2821 2024-09-18T01:00:45.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2820 2024-09-18T01:00:45.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:45.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:00:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:45.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:00:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:00:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:00:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:00:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:00:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:00:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:00:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:00:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:00:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:00:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:00:46.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2823 2024-09-18T01:00:46.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:47.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2826 2024-09-18T01:00:47.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2824 2024-09-18T01:00:47.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2825 2024-09-18T01:00:47.162 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:47.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:47.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:00:47.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:47.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:47.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:00:47.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:00:47.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:47.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:00:47.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:47.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:47.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:00:47.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:00:47.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:47.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:48.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2827 2024-09-18T01:00:48.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:48.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:48.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:00:48.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:48.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:48.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:00:48.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:00:48.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:00:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:00:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:00:48.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:48.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:00:48.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:48.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:48.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:00:48.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:00:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:49.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:00:49.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:49.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:49.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:00:49.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:00:49.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2828 2024-09-18T01:00:49.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2829 2024-09-18T01:00:49.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:49.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:50.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2831 2024-09-18T01:00:50.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:50.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:00:50.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:50.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:50.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:00:50.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:00:50.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2830 2024-09-18T01:00:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:50.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:50.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:00:50.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:50.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:50.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:00:50.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:00:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:00:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:00:50.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:00:50.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2832 2024-09-18T01:00:50.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:51.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:51.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:00:51.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:51.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:51.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:00:51.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:00:51.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2833 2024-09-18T01:00:51.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:52.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2834 2024-09-18T01:00:52.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2836 2024-09-18T01:00:52.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:52.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:52.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:00:52.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:00:52.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:52.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:52.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:52.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:52.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:00:52.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:00:52.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:00:52.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:00:52.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2835 2024-09-18T01:00:52.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:52.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2837 2024-09-18T01:00:53.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:53.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:53.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:00:53.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:53.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:53.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:00:53.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:00:53.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:53.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:00:53.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:53.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:53.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:00:53.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:00:53.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:53.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:00:53.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:53.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:53.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:00:53.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:00:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2838 2024-09-18T01:00:54.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:54.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:00:54.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:54.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:54.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:00:54.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:00:54.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2839 2024-09-18T01:00:54.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:54.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:55.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2840 2024-09-18T01:00:55.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2842 2024-09-18T01:00:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:00:55.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:55.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:55.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:00:55.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:00:55.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:55.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:00:55.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:55.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:55.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:00:55.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:00:55.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2841 2024-09-18T01:00:55.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:55.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:55.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:00:55.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:55.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:55.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:00:55.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:00:55.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:55.715 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:00:55.705+0000 7f52d7c6c640 -1 reset not still connected to 0x5578426955f0 2024-09-18T01:00:55.716 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:00:55.713+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f85b0 2024-09-18T01:00:55.717 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:00:55.713+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f89c0 2024-09-18T01:00:55.720 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:00:55.713+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T01:00:55.720 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:00:55.713+0000 7f52d7c6c640 -1 reset not still connected to 0x557848f21930 2024-09-18T01:00:55.720 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:00:55.713+0000 7f52d7c6c640 -1 reset not still connected to 0x5578517e9110 2024-09-18T01:00:55.721 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:00:55.713+0000 7f52d7c6c640 -1 reset not still connected to 0x5578577340d0 2024-09-18T01:00:55.721 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:00:55.713+0000 7f52d7c6c640 -1 reset not still connected to 0x5578591f5c70 2024-09-18T01:00:55.721 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:00:55.713+0000 7f52d7c6c640 -1 reset not still connected to 0x5578709dc1a0 2024-09-18T01:00:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:00:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:00:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:00:56.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2843 2024-09-18T01:00:56.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:57.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2845 2024-09-18T01:00:57.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2844 2024-09-18T01:00:57.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:57.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:00:57.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:57.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:57.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:00:57.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:00:57.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2846 2024-09-18T01:00:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:57.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:57.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:00:57.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:57.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:57.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:00:57.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:00:57.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:58.049 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2847 2024-09-18T01:00:58.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:58.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:58.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:00:58.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:58.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:58.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:00:58.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:00:58.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:58.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:00:58.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:58.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:58.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:00:58.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:00:58.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:58.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:00:58.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:58.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:58.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:00:58.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:00:59.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2848 2024-09-18T01:00:59.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2849 2024-09-18T01:00:59.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:00:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:00:59.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:00:59.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:00:59.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:00:59.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:00:59.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:01:00.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2852 2024-09-18T01:01:00.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2850 2024-09-18T01:01:00.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:00.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2851 2024-09-18T01:01:00.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:00.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:00.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:00.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:01:00.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:00.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:00.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:01:00.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:01:00.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:00.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:01:00.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:00.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:00.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:01:00.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:01:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:01:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:01:00.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:01:01.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2853 2024-09-18T01:01:01.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:01.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:01:01.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:01.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:01.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:01:01.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:01:01.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:01.925 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:01:01.920+0000 7f849c00d640 -1 reset not still connected to 0x5629d13be750 2024-09-18T01:01:01.936 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:01:01.932+0000 7f849c00d640 -1 reset not still connected to 0x5629d3685e10 2024-09-18T01:01:01.936 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:01:01.932+0000 7f849c00d640 -1 reset not still connected to 0x5629ed2bf1e0 2024-09-18T01:01:01.936 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:01:01.932+0000 7f849c00d640 -1 reset not still connected to 0x5629f3c5cc30 2024-09-18T01:01:02.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:02.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:01:02.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:02.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:02.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:01:02.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:01:02.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2855 2024-09-18T01:01:02.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2854 2024-09-18T01:01:02.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2856 2024-09-18T01:01:02.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:02.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:02.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:01:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:02.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:01:02.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:01:03.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2857 2024-09-18T01:01:03.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:03.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:03.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:01:03.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:03.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:03.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:01:03.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:01:03.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:03.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:01:03.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:03.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:03.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:01:03.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:01:03.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:03.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:01:03.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:03.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:03.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:01:03.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:01:04.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2858 2024-09-18T01:01:04.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:04.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:01:04.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:04.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:04.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:01:04.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:01:04.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2859 2024-09-18T01:01:04.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:04.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:04.129 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:01:04.124+0000 7f849c00d640 -1 reset not still connected to 0x5629f3c5cc30 2024-09-18T01:01:05.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2861 2024-09-18T01:01:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:01:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:01:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:01:05.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2860 2024-09-18T01:01:05.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2862 2024-09-18T01:01:05.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:05.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:01:05.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:05.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:05.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:01:05.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:01:05.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:01:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:01:05.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:01:05.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:05.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:05.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:01:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:01:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:01:06.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2863 2024-09-18T01:01:06.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:07.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2866 2024-09-18T01:01:07.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2865 2024-09-18T01:01:07.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:07.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:01:07.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:07.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:07.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:01:07.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:01:07.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:07.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:01:07.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:07.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:07.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:01:07.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:01:07.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2864 2024-09-18T01:01:07.118 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:07.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:07.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:08.048 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2867 2024-09-18T01:01:08.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:08.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:08.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:01:08.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:08.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:08.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:01:08.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:01:08.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:08.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:01:08.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:08.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:08.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:01:08.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:01:08.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:08.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:01:08.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:08.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:08.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:01:08.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:01:08.826 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:08.821+0000 7f52d7c6c640 -1 reset not still connected to 0x5578591f5c70 2024-09-18T01:01:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2869 2024-09-18T01:01:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:09.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:01:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:01:09.101 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:01:09.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2868 2024-09-18T01:01:09.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:09.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:10.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2872 2024-09-18T01:01:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2871 2024-09-18T01:01:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2870 2024-09-18T01:01:10.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:10.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:10.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:01:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:01:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:01:10.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:10.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:10.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:01:10.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:10.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:10.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:01:10.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:01:10.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:10.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:01:10.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:10.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:10.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:01:10.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:01:10.728 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:10.721+0000 7f52d7c6c640 -1 reset not still connected to 0x5578426955f0 2024-09-18T01:01:10.728 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:10.721+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T01:01:10.729 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:10.721+0000 7f52d7c6c640 -1 reset not still connected to 0x557848f21930 2024-09-18T01:01:10.729 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:10.721+0000 7f52d7c6c640 -1 reset not still connected to 0x5578517e9110 2024-09-18T01:01:10.729 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:10.721+0000 7f52d7c6c640 -1 reset not still connected to 0x5578577340d0 2024-09-18T01:01:10.729 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:10.725+0000 7f52d7c6c640 -1 reset not still connected to 0x5578591f5c70 2024-09-18T01:01:10.731 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:10.725+0000 7f52d7c6c640 -1 reset not still connected to 0x5578709dc1a0 2024-09-18T01:01:11.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:11.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:01:11.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:11.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:11.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:01:11.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:01:11.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2873 2024-09-18T01:01:11.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2875 2024-09-18T01:01:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:01:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:01:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:01:12.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2876 2024-09-18T01:01:12.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2874 2024-09-18T01:01:12.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:12.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:01:12.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:12.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:12.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:01:12.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:01:12.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:12.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:12.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:12.929 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:12.925+0000 7f52d7c6c640 -1 reset not still connected to 0x5578517e9110 2024-09-18T01:01:12.932 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:12.925+0000 7f52d7c6c640 -1 reset not still connected to 0x5578577340d0 2024-09-18T01:01:12.932 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:12.925+0000 7f52d7c6c640 -1 reset not still connected to 0x5578591f5c70 2024-09-18T01:01:12.932 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:12.925+0000 7f52d7c6c640 -1 reset not still connected to 0x5578709dc1a0 2024-09-18T01:01:13.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2877 2024-09-18T01:01:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:01:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:01:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:01:13.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:13.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:13.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:01:13.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:13.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:13.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:01:13.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:01:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:01:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:01:13.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:01:14.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:14.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:01:14.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:14.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:14.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:01:14.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:01:14.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2878 2024-09-18T01:01:14.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2879 2024-09-18T01:01:14.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:14.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:01:15.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2881 2024-09-18T01:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:15.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:01:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:01:15.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:15.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:01:15.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:15.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:15.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:01:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:01:15.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2882 2024-09-18T01:01:15.168 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2880 2024-09-18T01:01:15.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:15.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:15.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:16.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:16.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:01:16.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:16.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:16.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:01:16.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:01:16.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2883 2024-09-18T01:01:16.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:17.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2886 2024-09-18T01:01:17.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2884 2024-09-18T01:01:17.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:17.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:01:17.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:17.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:17.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:01:17.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:01:17.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2885 2024-09-18T01:01:17.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:17.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:01:17.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:17.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:01:17.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:01:17.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:17.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:17.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:18.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2887 2024-09-18T01:01:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:18.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:18.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:01:18.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:18.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:18.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:01:18.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:01:18.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:18.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:01:18.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:18.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:18.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:01:18.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:01:18.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:18.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:01:18.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:18.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:18.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:01:18.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:01:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2889 2024-09-18T01:01:19.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2888 2024-09-18T01:01:19.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:19.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:01:19.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:19.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:19.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:01:19.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:01:19.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:19.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:20.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2890 2024-09-18T01:01:20.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2892 2024-09-18T01:01:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2891 2024-09-18T01:01:20.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:20.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:20.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:01:20.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:20.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:20.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:01:20.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:01:20.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:20.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:01:20.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:20.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:20.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:01:20.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:01:20.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:20.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:20.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:01:20.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:20.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:20.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:01:20.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:01:20.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:01:21.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:21.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:21.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:01:21.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:01:21.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2893 2024-09-18T01:01:21.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:22.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2895 2024-09-18T01:01:22.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:22.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:01:22.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:22.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:22.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:01:22.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:01:22.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2896 2024-09-18T01:01:22.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:22.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:01:22.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:22.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:22.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:01:22.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:01:22.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:22.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:22.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2894 2024-09-18T01:01:22.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2897 2024-09-18T01:01:23.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:23.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:23.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:01:23.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:23.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:23.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:01:23.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:01:23.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:23.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:01:23.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:23.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:01:23.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:01:23.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:23.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:01:23.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:23.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:23.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:01:23.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:01:24.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2899 2024-09-18T01:01:24.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2898 2024-09-18T01:01:24.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:24.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:01:24.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:24.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:24.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:01:24.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:01:24.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:24.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:25.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2901 2024-09-18T01:01:25.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2900 2024-09-18T01:01:25.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:25.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:25.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:01:25.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:25.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:25.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:01:25.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:01:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:25.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:25.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:01:25.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:25.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:25.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:01:25.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:01:25.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2902 2024-09-18T01:01:25.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:25.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:01:25.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:25.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:25.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:01:25.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:01:25.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:26.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:26.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:01:26.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:26.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:01:26.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:01:26.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2903 2024-09-18T01:01:26.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:27.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2904 2024-09-18T01:01:27.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2905 2024-09-18T01:01:27.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2906 2024-09-18T01:01:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:01:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:01:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:01:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:01:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:27.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:27.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:01:27.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:01:27.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:27.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:27.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:28.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2907 2024-09-18T01:01:28.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:28.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:28.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:01:28.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:28.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:28.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:01:28.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:01:28.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:28.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:01:28.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:28.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:28.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:01:28.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:01:28.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:28.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:01:28.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:28.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:28.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:01:28.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:01:29.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2908 2024-09-18T01:01:29.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2909 2024-09-18T01:01:29.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:29.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:29.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:29.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:01:29.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:29.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:29.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:01:29.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:01:30.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2911 2024-09-18T01:01:30.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2912 2024-09-18T01:01:30.080 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2910 2024-09-18T01:01:30.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:30.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:30.102 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:30.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:30.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:01:30.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:30.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:30.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:01:30.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:01:30.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:30.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:01:30.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:30.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:30.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:01:30.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:01:31.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:31.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:01:31.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:31.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:31.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:01:31.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:01:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2913 2024-09-18T01:01:31.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:32.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:32.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:01:32.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:32.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:32.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:01:32.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:01:32.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2914 2024-09-18T01:01:32.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:32.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:01:32.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:32.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:32.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:01:32.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:01:32.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2915 2024-09-18T01:01:32.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2916 2024-09-18T01:01:32.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:32.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:32.835 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:32.832+0000 7f52d7c6c640 -1 reset not still connected to 0x5578426955f0 2024-09-18T01:01:32.835 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:32.832+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f85b0 2024-09-18T01:01:32.836 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:32.832+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f89c0 2024-09-18T01:01:32.836 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:32.832+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T01:01:32.836 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:32.836+0000 7f52d7c6c640 -1 reset not still connected to 0x557848f21930 2024-09-18T01:01:32.836 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:32.836+0000 7f52d7c6c640 -1 reset not still connected to 0x5578517e9110 2024-09-18T01:01:32.836 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:32.836+0000 7f52d7c6c640 -1 reset not still connected to 0x5578577340d0 2024-09-18T01:01:32.836 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:32.836+0000 7f52d7c6c640 -1 reset not still connected to 0x5578591f5c70 2024-09-18T01:01:32.837 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:01:32.836+0000 7f52d7c6c640 -1 reset not still connected to 0x5578709dc1a0 2024-09-18T01:01:33.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2917 2024-09-18T01:01:33.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:33.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:01:33.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:33.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:33.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:01:33.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:01:33.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:01:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:01:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:01:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:01:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:01:33.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:01:34.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2919 2024-09-18T01:01:34.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:34.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:01:34.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:34.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:34.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:01:34.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:01:34.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2918 2024-09-18T01:01:34.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:34.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:35.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2921 2024-09-18T01:01:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2922 2024-09-18T01:01:35.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:35.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:35.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:01:35.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:35.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:35.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:01:35.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:01:35.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2920 2024-09-18T01:01:35.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:35.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:01:35.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:35.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:35.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:01:35.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:01:35.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:35.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:01:35.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:35.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:35.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:01:35.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:01:35.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:35.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:36.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:36.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:01:36.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:36.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:36.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:01:36.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:01:36.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2923 2024-09-18T01:01:36.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:37.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2924 2024-09-18T01:01:37.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2926 2024-09-18T01:01:37.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:37.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:01:37.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:37.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:37.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:01:37.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:01:37.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2925 2024-09-18T01:01:37.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:37.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:01:37.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:38.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2927 2024-09-18T01:01:38.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:38.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:01:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:01:38.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:01:38.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:38.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:01:38.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:38.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:38.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:01:38.135 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:01:38.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:38.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:01:38.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:38.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:38.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:01:38.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:01:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:01:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:01:39.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:01:39.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2929 2024-09-18T01:01:39.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2928 2024-09-18T01:01:39.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:39.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:40.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2931 2024-09-18T01:01:40.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2932 2024-09-18T01:01:40.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:40.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:40.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:01:40.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:01:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:01:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:01:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:01:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:01:40.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2930 2024-09-18T01:01:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:40.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:01:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:01:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:01:40.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:41.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:41.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:01:41.106 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:41.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:41.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:01:41.107 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:01:41.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2933 2024-09-18T01:01:41.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:42.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2936 2024-09-18T01:01:42.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2934 2024-09-18T01:01:42.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2935 2024-09-18T01:01:42.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:42.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:01:42.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:42.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:42.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:01:42.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:01:42.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:42.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:01:42.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:42.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:42.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:01:42.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:01:42.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:42.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:42.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:43.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2937 2024-09-18T01:01:43.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:01:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:01:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:01:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:01:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:01:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:01:43.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:01:43.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 119942 2024-09-18T01:01:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:01:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:01:44.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:01:44.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2939 2024-09-18T01:01:44.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:44.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2938 2024-09-18T01:01:44.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:45.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2941 2024-09-18T01:01:45.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:45.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:01:45.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:45.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:45.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:01:45.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:01:45.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2940 2024-09-18T01:01:45.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:45.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:45.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:01:45.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:45.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:45.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:01:45.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:01:45.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:01:45.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 120112 2024-09-18T01:01:45.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:01:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:01:46.108 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:01:46.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2942 2024-09-18T01:01:46.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:47.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:47.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:01:47.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:47.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:47.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:01:47.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:01:47.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2943 2024-09-18T01:01:47.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:47.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:01:47.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:47.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:47.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:01:47.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:01:47.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2944 2024-09-18T01:01:47.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:47.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:48.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2945 2024-09-18T01:01:48.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:48.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:01:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:01:48.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:01:48.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:01:48.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 120284 2024-09-18T01:01:49.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2946 2024-09-18T01:01:49.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2947 2024-09-18T01:01:49.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:49.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:01:49.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:49.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:49.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:01:49.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:01:49.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:49.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:50.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2948 2024-09-18T01:01:50.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:50.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:50.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:01:50.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:50.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:50.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:01:50.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:01:50.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:01:50.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 120419 2024-09-18T01:01:51.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:51.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:01:51.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:51.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:51.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:01:51.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:01:51.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2949 2024-09-18T01:01:51.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:01:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 120659 2024-09-18T01:01:52.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:52.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:01:52.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:52.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:52.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:01:52.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:01:52.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2950 2024-09-18T01:01:52.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:53.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2951 2024-09-18T01:01:53.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:53.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:53.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:01:53.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:53.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:53.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:01:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:01:54.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2952 2024-09-18T01:01:54.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:01:54.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 120818 2024-09-18T01:01:54.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:55.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2953 2024-09-18T01:01:55.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:55.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:55.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:01:55.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:55.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:55.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:01:55.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:01:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:01:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 121001 2024-09-18T01:01:57.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:01:57.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:01:57.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:01:57.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:01:57.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:01:57.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:01:57.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2954 2024-09-18T01:01:57.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:01:58.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:01:58.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 121218 2024-09-18T01:01:59.073 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:01:59.063+0000 7f849c00d640 -1 reset not still connected to 0x5629c077c5b0 2024-09-18T01:01:59.075 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:01:59.063+0000 7f849c00d640 -1 reset not still connected to 0x5629d13be750 2024-09-18T01:01:59.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2955 2024-09-18T01:01:59.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:02:00.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:02:00.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 121465 2024-09-18T01:02:02.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T01:02:02.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:02:02.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 119808 2024-09-18T01:04:16.124 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:04:16.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:04:16.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:04:16.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 119943 2024-09-18T01:04:17.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:04:17.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:04:17.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:04:17.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 120113 2024-09-18T01:04:19.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:04:19.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:04:19.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:04:19.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 120285 2024-09-18T01:04:21.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:04:21.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:04:21.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:04:21.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 120420 2024-09-18T01:04:23.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:04:23.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:04:23.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:04:23.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 120660 2024-09-18T01:04:25.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:04:25.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:04:25.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:04:25.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 120819 2024-09-18T01:04:27.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:04:27.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:04:27.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:04:27.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 121002 2024-09-18T01:04:29.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:04:29.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:04:29.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:04:29.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 121219 2024-09-18T01:04:31.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:04:31.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:04:31.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:04:31.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 121466 2024-09-18T01:04:33.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:04:33.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:04:33.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T01:04:33.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:04:33.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:04:33.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 1 2024-09-18T01:04:33.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:04:33.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd0 2024-09-18T01:04:33.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary1 2024-09-18T01:04:33.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary1 2024-09-18T01:04:33.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:04:33.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary1 2024-09-18T01:04:33.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:04:33.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:04:33.722 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 2 2024-09-18T01:04:33.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:04:33.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 3 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd1 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary2 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary2 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd2 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary3 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary3 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary2 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary3 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:04:33.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 4 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd3 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary4 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary4 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 5 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary4 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd4 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 6 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:04:33.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:04:33.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd5 2024-09-18T01:04:33.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary6 2024-09-18T01:04:33.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary6 2024-09-18T01:04:33.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:04:33.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary6 2024-09-18T01:04:33.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary5 2024-09-18T01:04:33.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:04:33.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary5 2024-09-18T01:04:33.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:04:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary5 2024-09-18T01:04:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:04:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 7 2024-09-18T01:04:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd6 2024-09-18T01:04:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:04:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary7 2024-09-18T01:04:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary7 2024-09-18T01:04:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:04:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary7 2024-09-18T01:04:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:04:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 8 2024-09-18T01:04:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd7 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary8 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary8 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary8 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 9 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd8 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary9 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary9 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary9 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:04:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 127328 2024-09-18T01:04:33.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 10 2024-09-18T01:04:33.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd9 2024-09-18T01:04:33.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary10 2024-09-18T01:04:33.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary10 2024-09-18T01:04:33.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:04:33.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary10 2024-09-18T01:04:34.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T01:04:34.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:04:34.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T01:04:34.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:04:34.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T01:04:34.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:04:34.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T01:04:34.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:04:34.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T01:04:34.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:04:34.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T01:04:34.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:04:34.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:04:34.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T01:04:34.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:04:34.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T01:04:34.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:04:34.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T01:04:34.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T01:04:34.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:05:32.628 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=4c7b33454fa589640611bf920524c548 2024-09-18T01:05:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary4 2024-09-18T01:05:33.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=0ac0647b23a0a22593b36a5af1fbf87d 2024-09-18T01:05:33.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary9 2024-09-18T01:05:34.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary4 2024-09-18T01:05:34.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:34.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:34.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:05:34.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary4 2024-09-18T01:05:34.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=da62857985a5ac35c60d2c0af4f221a2 2024-09-18T01:05:34.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary2 2024-09-18T01:05:34.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=c2c7e66cf17fe9eab5b03dfd6d3e1544 2024-09-18T01:05:34.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary7 2024-09-18T01:05:35.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=d51bd2c02ece36058b8b796ebab9a026 2024-09-18T01:05:35.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary5 2024-09-18T01:05:35.179 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary9 2024-09-18T01:05:35.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:35.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:35.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:05:35.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary9 2024-09-18T01:05:35.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:05:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+unknown 2024-09-18T01:05:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:05:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:35.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary2 2024-09-18T01:05:35.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:35.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:35.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:05:35.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary2 2024-09-18T01:05:35.990 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary7 2024-09-18T01:05:35.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:35.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:35.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:05:35.991 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary7 2024-09-18T01:05:36.250 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-18T01:05:36.233+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0dabb790 2024-09-18T01:05:36.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=aac76d074651b3a822b652766de75dfd 2024-09-18T01:05:36.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary6 2024-09-18T01:05:36.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary5 2024-09-18T01:05:36.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:36.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:36.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:05:36.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary5 2024-09-18T01:05:36.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=f0847e53a152267399727e77a25c84dc 2024-09-18T01:05:36.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary1 2024-09-18T01:05:36.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T01:05:36.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:36.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:36.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:05:36.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:36.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:36.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:36.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:36.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:36.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T01:05:36.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:05:36.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=e8dd5e23bd56fc9d103f5e9a487ea87a 2024-09-18T01:05:36.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary10 2024-09-18T01:05:36.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+unknown 2024-09-18T01:05:36.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:36.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:36.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:05:36.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:36.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:36.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:36.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:36.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T01:05:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f16bbff5-762b-414e-93aa-bcfe90e66fb3 2024-09-18T01:05:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:05:36.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:05:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:34 2024-09-18T01:05:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:36' 2024-09-18T01:05:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:36.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:05:36.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:36.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T01:05:37.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary6 2024-09-18T01:05:37.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:37.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:37.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:05:37.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary6 2024-09-18T01:05:37.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary1 2024-09-18T01:05:37.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:37.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:37.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:05:37.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary1 2024-09-18T01:05:37.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=6dd9fd402157d14a8d9bce3d0d6c93c9 2024-09-18T01:05:37.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary8 2024-09-18T01:05:37.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=ccd2ed62827b8ec71f304ff9f1a80a55 2024-09-18T01:05:37.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary3 2024-09-18T01:05:37.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary10 2024-09-18T01:05:37.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:37.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:37.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:05:37.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary10 2024-09-18T01:05:37.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:05:37.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:05:37.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T01:05:37.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:37.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:37.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:05:37.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:37.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:37.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T01:05:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:05:37.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+unknown 2024-09-18T01:05:37.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:37.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:37.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:05:37.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:37.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:37.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:37.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:37.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:37.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+unknown 2024-09-18T01:05:37.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:05:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+unknown 2024-09-18T01:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T01:05:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 36b71294-3a4d-45ce-bfdd-d89c2dfd7337 2024-09-18T01:05:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:05:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:05:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:34 2024-09-18T01:05:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:38.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:38.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:37' 2024-09-18T01:05:38.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:38.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T01:05:38.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary3 2024-09-18T01:05:38.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:38.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:38.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:05:38.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary3 2024-09-18T01:05:38.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary8 2024-09-18T01:05:38.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:38.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:38.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:05:38.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary8 2024-09-18T01:05:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:05:38.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:05:38.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T01:05:38.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:38.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:38.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:05:38.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:38.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:38.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:38.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:38.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:38.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T01:05:38.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T01:05:38.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:38.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:38.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:05:38.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:38.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:38.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:38.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:38.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:38.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T01:05:38.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+unknown 2024-09-18T01:05:38.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:38.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:38.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:05:38.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:38.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:38.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:38.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:38.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:38.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+unknown 2024-09-18T01:05:38.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:38.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:38.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:05:38.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:38.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:38.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:38.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:38.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:38.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T01:05:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:05:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:38.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T01:05:38.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T01:05:38.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:38.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:38.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:05:38.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:38.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:38.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:38.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:38.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:38.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T01:05:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T01:05:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 5984c013-9b62-4bf1-bc42-f8038f39f85c 2024-09-18T01:05:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:05:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:05:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:34 2024-09-18T01:05:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:38.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:38.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:38.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:38' 2024-09-18T01:05:38.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:38.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:05:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T01:05:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T01:05:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 640308f0-3a45-4793-82ba-ab0e29660b48 2024-09-18T01:05:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:05:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:05:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:34 2024-09-18T01:05:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:38' 2024-09-18T01:05:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:39.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:05:39.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:39.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T01:05:39.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T01:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 225f2829-01d3-45e2-a1d6-0f790148e4c3 2024-09-18T01:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:34 2024-09-18T01:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:38' 2024-09-18T01:05:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:39.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:05:39.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:39.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T01:05:39.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T01:05:39.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f16bbff5-762b-414e-93aa-bcfe90e66fb3 2024-09-18T01:05:39.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:05:39.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:05:39.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:39.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:34 2024-09-18T01:05:39.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:39.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:39.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:39.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:39.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:36' 2024-09-18T01:05:39.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:39.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:05:39.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:39.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T01:05:39.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:05:39.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:05:39.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:05:39.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+unknown 2024-09-18T01:05:39.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:39.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:39.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:05:39.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:39.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:39.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:39.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:39.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:39.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+unknown 2024-09-18T01:05:39.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:39.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:39.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:05:39.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:39.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:39.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:39.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:39.907 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+unknown 2024-09-18T01:05:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:05:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:39.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T01:05:39.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:39.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:39.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:05:39.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:39.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:39.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:39.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:39.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:39.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T01:05:39.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T01:05:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:05:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:39.944 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T01:05:39.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T01:05:39.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 97f2fe38-61d0-453f-b29c-085ba52e905a 2024-09-18T01:05:39.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:39.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:39.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:39.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:39.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:39.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:39.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:39.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:39.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:39.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:39.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:39.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:40.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:40.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:40.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:40.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:40.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:40.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T01:05:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a61c5b8e-bc8f-4d8e-b48f-991b9b77e317 2024-09-18T01:05:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:40.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:40.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:40.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:40.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:40.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:40.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:40.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:40.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:40.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T01:05:40.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:40.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:40.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:40.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:05:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T01:05:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:40.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:40.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:40.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:40.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:05:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:05:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:40.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:40.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T01:05:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 36b71294-3a4d-45ce-bfdd-d89c2dfd7337 2024-09-18T01:05:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:40.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:40.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:40.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:40.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:40.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:40.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:40.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:40.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:40.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:40.093 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:05:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2134016.0,"last_snapshot_bytes":1687552,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621318,"remote_snapshot_timestamp":1726621318,"replay_state":"idle"}2024-09-18 01:05:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":182272.0,"last_snapshot_bytes":139264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621304,"remote_snapshot_timestamp":1726621304,"replay_state":"idle"}2024-09-18 01:05:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":7395328.0,"last_snapshot_bytes":6766592,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621314,"remote_snapshot_timestamp":1726621314,"replay_state":"idle"}2024-09-18 01:05:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2134016.0,"last_snapshot_bytes":1687552,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621318,"remote_snapshot_timestamp":1726621318,"replay_state":"idle"}2024-09-18 01:05:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":182272.0,"last_snapshot_bytes":139264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621304,"remote_snapshot_timestamp":1726621304,"replay_state":"idle"}2024-09-18 01:05:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":7395328.0,"last_snapshot_bytes":6766592,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621314,"remote_snapshot_timestamp":1726621314,"replay_state":"idle"}2024-09-18 01:05:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:40.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:40.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T01:05:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T01:05:40.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 01:05:40image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 01:05:40image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 01:05:40image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:40.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:40.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T01:05:40.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:40.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T01:05:40.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T01:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T01:05:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:40.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:40.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:40.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:40.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:40.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:40.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:40.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:40.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:40.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:40.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:40.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:40.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:40.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary1 up+unknown 2024-09-18T01:05:40.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:40.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:40.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:40.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:05:40.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:40.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:40.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:40.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:40.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:40.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary6 up+unknown 2024-09-18T01:05:40.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:40.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:40.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:40.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:05:40.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:40.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:40.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:40.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:40.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:40.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:40.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:40.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary9 up+unknown 2024-09-18T01:05:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:05:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:40.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T01:05:40.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:40.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:40.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:05:40.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T01:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T01:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T01:05:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T01:05:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:05:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:40.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:40.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T01:05:40.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T01:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 943aeb36-cdea-4614-a073-08041a451b48 2024-09-18T01:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:40' 2024-09-18T01:05:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:40.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:40.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:40.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:40.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T01:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T01:05:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:05:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f1cbc639-19c2-4066-bbe2-5861c8b3f55a 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:40' 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T01:05:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:05:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T01:05:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:41.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:41.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T01:05:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:05:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T01:05:41.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:41.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:41.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:05:41.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:41.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T01:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4a901bdb-c18f-43d9-a0f0-efe4d94ef3aa 2024-09-18T01:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:40' 2024-09-18T01:05:41.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:41.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:41.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:41.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:41.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:41.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:41.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:41.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:41.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:05:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T01:05:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 5984c013-9b62-4bf1-bc42-f8038f39f85c 2024-09-18T01:05:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:41.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:41.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:41.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:41.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:41.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T01:05:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 225f2829-01d3-45e2-a1d6-0f790148e4c3 2024-09-18T01:05:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:41.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:41.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:41.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:41.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:41.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:41.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:41.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:41.141 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:41.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:41.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:41.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:41.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:41.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:05:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:05:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T01:05:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 640308f0-3a45-4793-82ba-ab0e29660b48 2024-09-18T01:05:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:41.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:41.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:41.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:41.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:41.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:41.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:41.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:41.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:41.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:41.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:41.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:41.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:41.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:05:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:41.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:41.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:41.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T01:05:41.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T01:05:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary10 up+unknown 2024-09-18T01:05:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:05:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:41.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:41.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:41.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T01:05:41.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T01:05:41.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:41.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:41.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary3 up+unknown 2024-09-18T01:05:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:05:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:41.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:41.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary1 up+unknown '' '' 2024-09-18T01:05:41.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary6 up+unknown '' '' 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:41.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary1 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary6 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary9 up+unknown '' '' 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary9 2024-09-18T01:05:41.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:41.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:41.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T01:05:41.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T01:05:41.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:41.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:41.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary7 up+unknown 2024-09-18T01:05:41.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:05:41.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:41.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:41.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:41.308 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:41.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:41.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:41.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T01:05:41.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T01:05:41.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:41.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:41.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T01:05:41.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:41.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:41.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary5 up+unknown 2024-09-18T01:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T01:05:41.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:41.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:41.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary2 up+unknown 2024-09-18T01:05:41.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:05:41.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:41.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:41.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:41.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-+ status='image-primary1: 2024-09-18T01:05:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a61c5b8e-bc8f-4d8e-b48f-991b9b77e317 2024-09-18T01:05:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:41.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:05:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:41.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:41.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:41.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:41.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T01:05:41.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:41.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:41.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T01:05:41.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:41.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:41.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:41.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T01:05:41.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T01:05:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 36b71294-3a4d-45ce-bfdd-d89c2dfd7337 2024-09-18T01:05:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:05:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:41.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:41.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary1 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:05:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:41.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:41.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:41.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T01:05:41.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 97f2fe38-61d0-453f-b29c-085ba52e905a 2024-09-18T01:05:41.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:41.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:41.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:41.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:41.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:05:41.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:41.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:41.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:41.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:41.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:41.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary8 up+unknown 2024-09-18T01:05:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:05:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:41.374 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:41.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T01:05:41.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:41.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:41.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:41.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:41.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:41.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:41.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary9 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:41.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:05:41.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:41.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:41.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:41.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:41.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:41.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary6 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:41.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:05:41.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:41.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:41.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:41.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:41.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:41.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:41.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:41.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T01:05:41.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:41.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:41.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T01:05:41.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T01:05:41.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T01:05:41.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:41.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:41.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:41.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T01:05:41.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:41.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:41.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary1 2024-09-18T01:05:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:05:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:05:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary1 2024-09-18T01:05:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary9 2024-09-18T01:05:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:05:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:05:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary9 2024-09-18T01:05:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:41.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T01:05:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:41.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:41.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:41.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:41.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary6 2024-09-18T01:05:41.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:41.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:41.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:05:41.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:05:41.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary6 2024-09-18T01:05:42.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary10 up+unknown '' '' 2024-09-18T01:05:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:05:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:42.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary10 2024-09-18T01:05:42.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary3 up+unknown '' '' 2024-09-18T01:05:42.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:05:42.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:42.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:42.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:42.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:42.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:42.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary3 2024-09-18T01:05:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:05:42.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:05:42.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:05:42.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:05:42.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T01:05:42.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:05:42.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T01:05:42.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:05:42.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T01:05:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T01:05:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 943aeb36-cdea-4614-a073-08041a451b48 2024-09-18T01:05:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:42.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:40 2024-09-18T01:05:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:05:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:42.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:42.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:42.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T01:05:42.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:42.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:42.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:42.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:42.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:42.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:42.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:42.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary7 up+unknown '' '' 2024-09-18T01:05:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:05:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary7 2024-09-18T01:05:42.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:40' 2024-09-18T01:05:42.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary10 up+unknown '2024-09-18 01:05:40' 2024-09-18T01:05:42.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:05:42.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:42.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:40' 2024-09-18T01:05:42.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:42.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:42.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:42.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T01:05:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4a901bdb-c18f-43d9-a0f0-efe4d94ef3aa 2024-09-18T01:05:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:40 2024-09-18T01:05:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:05:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T01:05:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:42.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:42.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:42.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:42.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:42.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:40' 2024-09-18T01:05:42.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary3 up+unknown '2024-09-18 01:05:40' 2024-09-18T01:05:42.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:05:42.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:42.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:40' 2024-09-18T01:05:42.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:42.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:42.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:42.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary5 up+unknown '' '' 2024-09-18T01:05:42.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:05:42.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:42.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:42.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:42.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:42.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:42.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary5 2024-09-18T01:05:42.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary2 up+unknown '' '' 2024-09-18T01:05:42.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:05:42.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:42.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:42.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:42.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:42.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:42.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary2 2024-09-18T01:05:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary8 up+unknown '' '' 2024-09-18T01:05:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:05:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:42.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:42.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:42.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary8 2024-09-18T01:05:42.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T01:05:42.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 225f2829-01d3-45e2-a1d6-0f790148e4c3 2024-09-18T01:05:42.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:42.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:42.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:42.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:42.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:42.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:05:42.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:42.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:42.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:42.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:42.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:42.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:42.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:42.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:42.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:42.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:42.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:42.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:42.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:42.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:42.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:42.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:42.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary7 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:42.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:05:42.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:42.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:42.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:42.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:42.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:42.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd0 2024-09-18T01:05:42.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T01:05:42.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:05:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T01:05:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f1cbc639-19c2-4066-bbe2-5861c8b3f55a 2024-09-18T01:05:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:05:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:40 2024-09-18T01:05:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:42.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:05:42.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:42.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:42.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:42.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:42.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:42.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:42.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T01:05:42.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:42.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:42.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:42.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:42.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:42.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd1 2024-09-18T01:05:42.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:42.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:42.511 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:42.519 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:05:42.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:40' 2024-09-18T01:05:42.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary8 up+unknown '2024-09-18 01:05:40' 2024-09-18T01:05:42.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:05:42.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:42.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:40' 2024-09-18T01:05:42.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:42.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:42.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:42.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T01:05:42.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T01:05:42.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 640308f0-3a45-4793-82ba-ab0e29660b48 2024-09-18T01:05:42.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:42.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:42.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:42.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:42.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:42.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:05:42.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:42.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:42.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:42.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:42.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:42.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:42.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:42.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:42.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:42.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:42.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:42.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:42.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:42.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:42.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:42.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary2 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:05:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:42.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:42.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:42.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd2 2024-09-18T01:05:42.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T01:05:42.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T01:05:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:40' 2024-09-18T01:05:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T01:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:42.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:42.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T01:05:42.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:40' 2024-09-18T01:05:42.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T01:05:42.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:42.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:42.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:42.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:42.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary10 2024-09-18T01:05:42.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:05:42.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:05:42.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary10 2024-09-18T01:05:42.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:42.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:42.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:42.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:42.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:42.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:42.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary3 2024-09-18T01:05:42.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:05:42.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:05:42.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary3 2024-09-18T01:05:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T01:05:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 5984c013-9b62-4bf1-bc42-f8038f39f85c 2024-09-18T01:05:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:05:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:42.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:42.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:42.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:42.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:42.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:42.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T01:05:42.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:42.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:42.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:42.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:42.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T01:05:42.755 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:42.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:42.756 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:42.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:42.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary5 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:42.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:05:42.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:42.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:42.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:42.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:42.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:42.760 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:42.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary7 2024-09-18T01:05:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:05:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:05:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary7 2024-09-18T01:05:42.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:42.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:42.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T01:05:42.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:42.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T01:05:42.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:42.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:42.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:42.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:42.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:42.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:42.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary2 2024-09-18T01:05:42.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:42.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:42.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:05:42.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:05:42.861 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary2 2024-09-18T01:05:43.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:43.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:43.022 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T01:05:43.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:40' 2024-09-18T01:05:43.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T01:05:43.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:43.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:43.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:43.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:43.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:43.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:43.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary8 2024-09-18T01:05:43.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:43.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:43.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:05:43.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:05:43.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary8 2024-09-18T01:05:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T01:05:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:05:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:43.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:43.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:43.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T01:05:43.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:05:40cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:43.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:43.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T01:05:43.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:43.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T01:05:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:43.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:43.153 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:43.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:43.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:43.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary5 2024-09-18T01:05:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:05:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:05:43.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary5 2024-09-18T01:05:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T01:05:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f16bbff5-762b-414e-93aa-bcfe90e66fb3 2024-09-18T01:05:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39 2024-09-18T01:05:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:05:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:39' 2024-09-18T01:05:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:43.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:43.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:43.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:43.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:43.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:43.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:43.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:43.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:43.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:43.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:43.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:43.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+unknown '2024-09-18 01:05:39' 2024-09-18T01:05:43.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:05:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:05:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:39' 2024-09-18T01:05:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:43.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:05:43.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:05:43.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:05:43.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:05:43.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T01:05:43.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:05:43.285 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:05:43.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:05:43.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:05:43.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T01:05:43.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:05:43.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T01:05:43.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:05:43.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T01:05:43.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T01:05:43.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:05:40image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:05:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:05:39' 2024-09-18T01:05:43.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:43.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T01:05:43.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:39' 2024-09-18T01:05:43.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T01:05:43.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:43.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:43.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:43.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:43.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:43.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:43.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary4 up+unknown 2024-09-18T01:05:43.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:43.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:43.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:05:43.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:43.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:43.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:05:43.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd3 2024-09-18T01:05:43.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T01:05:43.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:05:43.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd4 2024-09-18T01:05:43.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T01:05:43.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:05:43.613 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd5 2024-09-18T01:05:43.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T01:05:43.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:05:43.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd6 2024-09-18T01:05:43.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T01:05:43.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:05:43.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd7 2024-09-18T01:05:43.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T01:05:43.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:05:44.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:05:44.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:05:44.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T01:05:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary4 up+unknown '' '' 2024-09-18T01:05:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:44.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:05:44.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:44.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:05:44.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:05:44.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:44.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:44.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary4 2024-09-18T01:05:44.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T01:05:44.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f16bbff5-762b-414e-93aa-bcfe90e66fb3 2024-09-18T01:05:44.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:44.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:44.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:05:44.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:44 2024-09-18T01:05:44.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:05:44.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:05:44.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:05:44.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:05:44.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:05:44' 2024-09-18T01:05:44.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:05:44.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:05:44.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:05:44.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:05:44.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:05:44.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:05:44.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:44.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:44.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:05:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:05:44.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:44' 2024-09-18T01:05:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary4 up+unknown '2024-09-18 01:05:44' 2024-09-18T01:05:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:05:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:05:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:05:44' 2024-09-18T01:05:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:05:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:05:44.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:44.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd8 2024-09-18T01:05:44.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:05:44.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T01:05:45.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING1942614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:05:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}2024-09-18 01:05:44image-primary10943aeb36-cdea-4614-a073-08041a451b48up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:05:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}2024-09-18 01:05:44image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:05:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}2024-09-18 01:05:44image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:05:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}2024-09-18 01:05:44image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:05:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:05:44image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:05:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621544,"replay_state":"idle"}2024-09-18 01:05:44image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:05:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}2024-09-18 01:05:44image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:05:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}2024-09-18 01:05:44image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:05:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}2024-09-18 01:05:44image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:05:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}2024-09-18 01:05:44' 2024-09-18T01:05:45.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:05:45.147 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T01:05:45.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:05:44' 2024-09-18T01:05:45.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T01:05:45.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:05:45.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:05:45.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:05:45.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:05:45.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:45.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:05:45.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary4 2024-09-18T01:05:45.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:05:45.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:05:45.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:05:45.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:05:45.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary4 2024-09-18T01:05:45.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:05:45.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:05:45.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T01:05:46.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd9 2024-09-18T01:05:46.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T01:05:46.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:06:50.037 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-18T01:06:50.035+0000 7fbbe8b19640 -1 reset not still connected to 0x561a1e8f4750 2024-09-18T01:06:56.742 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:06:56.739+0000 7f52d7c6c640 -1 reset not still connected to 0x557875a09ad0 2024-09-18T01:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=aac76d074651b3a822b652766de75dfd 2024-09-18T01:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd1 2024-09-18T01:07:34.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ aac76d074651b3a822b652766de75dfd != \a\a\c\7\6\d\0\7\4\6\5\1\b\3\a\8\2\2\b\6\5\2\7\6\6\d\e\7\5\d\f\d ]] 2024-09-18T01:07:34.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=ccd2ed62827b8ec71f304ff9f1a80a55 2024-09-18T01:07:34.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd5 2024-09-18T01:07:34.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=6dd9fd402157d14a8d9bce3d0d6c93c9 2024-09-18T01:07:34.921 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd3 2024-09-18T01:07:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=f0847e53a152267399727e77a25c84dc 2024-09-18T01:07:35.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd2 2024-09-18T01:07:35.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ ccd2ed62827b8ec71f304ff9f1a80a55 != \c\c\d\2\e\d\6\2\8\2\7\b\8\e\c\7\1\f\3\0\4\f\f\9\f\1\a\8\0\a\5\5 ]] 2024-09-18T01:07:36.014 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 6dd9fd402157d14a8d9bce3d0d6c93c9 != \6\d\d\9\f\d\4\0\2\1\5\7\d\1\4\a\8\d\9\b\c\e\3\d\0\d\6\c\9\3\c\9 ]] 2024-09-18T01:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=da62857985a5ac35c60d2c0af4f221a2 2024-09-18T01:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd7 2024-09-18T01:07:36.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ f0847e53a152267399727e77a25c84dc != \f\0\8\4\7\e\5\3\a\1\5\2\2\6\7\3\9\9\7\2\7\e\7\7\a\2\5\c\8\4\d\c ]] 2024-09-18T01:07:36.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:07:36.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 127329 2024-09-18T01:07:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=e8dd5e23bd56fc9d103f5e9a487ea87a 2024-09-18T01:07:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd4 2024-09-18T01:07:37.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ da62857985a5ac35c60d2c0af4f221a2 != \d\a\6\2\8\5\7\9\8\5\a\5\a\c\3\5\c\6\0\d\2\c\0\a\f\4\f\2\2\1\a\2 ]] 2024-09-18T01:07:37.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:07:37.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 127331 2024-09-18T01:07:37.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:07:37.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 127332 2024-09-18T01:07:37.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=0ac0647b23a0a22593b36a5af1fbf87d 2024-09-18T01:07:37.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd0 2024-09-18T01:07:37.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=c2c7e66cf17fe9eab5b03dfd6d3e1544 2024-09-18T01:07:37.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd6 2024-09-18T01:07:37.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ e8dd5e23bd56fc9d103f5e9a487ea87a != \e\8\d\d\5\e\2\3\b\d\5\6\f\c\9\d\1\0\3\f\5\e\9\a\4\8\7\e\a\8\7\a ]] 2024-09-18T01:07:37.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=d51bd2c02ece36058b8b796ebab9a026 2024-09-18T01:07:37.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd8 2024-09-18T01:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 0ac0647b23a0a22593b36a5af1fbf87d != \0\a\c\0\6\4\7\b\2\3\a\0\a\2\2\5\9\3\b\3\6\a\5\a\f\1\f\b\f\8\7\d ]] 2024-09-18T01:07:37.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=4c7b33454fa589640611bf920524c548 2024-09-18T01:07:37.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd9 2024-09-18T01:07:38.031 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ c2c7e66cf17fe9eab5b03dfd6d3e1544 != \c\2\c\7\e\6\6\c\f\1\7\f\e\9\e\a\b\5\b\0\3\d\f\d\6\d\3\e\1\5\4\4 ]] 2024-09-18T01:07:38.411 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ d51bd2c02ece36058b8b796ebab9a026 != \d\5\1\b\d\2\c\0\2\e\c\e\3\6\0\5\8\b\8\b\7\9\6\e\b\a\b\9\a\0\2\6 ]] 2024-09-18T01:07:38.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 4c7b33454fa589640611bf920524c548 != \4\c\7\b\3\3\4\5\4\f\a\5\8\9\6\4\0\6\1\1\b\f\9\2\0\5\2\4\c\5\4\8 ]] 2024-09-18T01:07:38.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:07:38.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 127335 2024-09-18T01:07:38.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:07:38.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 127337 2024-09-18T01:07:38.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:07:38.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 127338 2024-09-18T01:07:38.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:07:38.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 127341 2024-09-18T01:07:38.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:07:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 127343 2024-09-18T01:07:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:07:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 127345 2024-09-18T01:07:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T01:07:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:07:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T01:07:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary1 2024-09-18T01:07:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:07:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:07:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+replaying 2024-09-18T01:07:38.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:38.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:38.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:07:38.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:38.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:38.452 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:38.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:38.453 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+replaying '' '' 2024-09-18T01:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:39.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:07:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:07:39.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T01:07:39.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T01:07:39.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: a61c5b8e-bc8f-4d8e-b48f-991b9b77e317 2024-09-18T01:07:39.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:07:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64258048.0,"last_snapshot_bytes":64258048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"} 2024-09-18T01:07:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:07:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:39 2024-09-18T01:07:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:07:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:07:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:07:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:07:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:39' 2024-09-18T01:07:39.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:07:39.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:07:39.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:07:39.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64258048.0,"last_snapshot_bytes":64258048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"} 2024-09-18T01:07:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:07:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:07:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:07:39.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:07:39.573 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:07:39.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:07:39.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:07:39.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:07:39.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:07:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+replaying '2024-09-18 01:07:39' 2024-09-18T01:07:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:07:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:07:39' 2024-09-18T01:07:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:07:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:07:39.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:07:39.694 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:39.688+0000 7f849c00d640 -1 reset not still connected to 0x5629e9ce5ba0 2024-09-18T01:07:39.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1a61c5b8e-bc8f-4d8e-b48f-991b9b77e317up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64258048.0,"last_snapshot_bytes":64258048,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary10943aeb36-cdea-4614-a073-08041a451b48up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70389760.0,"last_snapshot_bytes":70389760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72048640.0,"last_snapshot_bytes":72048640,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64069632.0,"last_snapshot_bytes":64069632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621545,"remote_snapshot_timestamp":1726621545,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":133029888.0,"last_snapshot_bytes":133029888,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621544,"remote_snapshot_timestamp":1726621544,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70172672.0,"last_snapshot_bytes":70172672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70217728.0,"last_snapshot_bytes":70217728,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70197248.0,"last_snapshot_bytes":70197248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70287360.0,"last_snapshot_bytes":70287360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39' 2024-09-18T01:07:39.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:07:39.745 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T01:07:39.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:07:39.812 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T01:07:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:07:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:07:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:07:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:07:39.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:39.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:39.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary1 2024-09-18T01:07:39.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:07:39.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:39.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:07:39.820 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary1 2024-09-18T01:07:39.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary1 2024-09-18T01:07:41.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:07:41.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary1 2024-09-18T01:07:41.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:07:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:07:41.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary1 2024-09-18T01:07:41.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:41.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary1 2024-09-18T01:07:41.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:07:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T01:07:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary2 2024-09-18T01:07:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:07:41.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:07:41.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+replaying 2024-09-18T01:07:41.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:41.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:41.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:07:41.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:41.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:41.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:07:42.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+replaying '' '' 2024-09-18T01:07:42.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:42.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:42.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:07:42.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:42.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:42.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:42.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:07:42.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:07:42.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T01:07:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T01:07:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 640308f0-3a45-4793-82ba-ab0e29660b48 2024-09-18T01:07:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:07:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72048640.0,"last_snapshot_bytes":72048640,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"} 2024-09-18T01:07:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:07:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:39 2024-09-18T01:07:43.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:07:43.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:07:43.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:07:43.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:07:43.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:39' 2024-09-18T01:07:43.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:07:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:07:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:07:43.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72048640.0,"last_snapshot_bytes":72048640,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"} 2024-09-18T01:07:43.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:07:43.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:07:43.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:07:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:07:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:07:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:07:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:07:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:07:43.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:07:43.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+replaying '2024-09-18 01:07:39' 2024-09-18T01:07:43.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:43.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:43.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:07:43.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:43.170 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:07:39' 2024-09-18T01:07:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:07:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:07:43.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:07:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10943aeb36-cdea-4614-a073-08041a451b48up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70389760.0,"last_snapshot_bytes":70389760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary2640308f0-3a45-4793-82ba-ab0e29660b48up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72048640.0,"last_snapshot_bytes":72048640,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64069632.0,"last_snapshot_bytes":64069632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621545,"remote_snapshot_timestamp":1726621545,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":133029888.0,"last_snapshot_bytes":133029888,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621544,"remote_snapshot_timestamp":1726621544,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70172672.0,"last_snapshot_bytes":70172672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70217728.0,"last_snapshot_bytes":70217728,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70197248.0,"last_snapshot_bytes":70197248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70287360.0,"last_snapshot_bytes":70287360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:39' 2024-09-18T01:07:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:07:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T01:07:43.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:07:43.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T01:07:43.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:07:43.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:07:43.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:07:43.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:07:43.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:43.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:43.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary2 2024-09-18T01:07:43.344 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:07:43.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:43.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:07:43.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary2 2024-09-18T01:07:43.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary2 2024-09-18T01:07:43.827 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:43.820+0000 7f849c00d640 -1 reset not still connected to 0x5629f09b11e0 2024-09-18T01:07:44.047 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:44.040+0000 7f849c00d640 -1 reset not still connected to 0x5629f09b1040 2024-09-18T01:07:44.047 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:44.040+0000 7f849c00d640 -1 reset not still connected to 0x5629f09b1110 2024-09-18T01:07:44.047 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:44.040+0000 7f849c00d640 -1 reset not still connected to 0x5629f09b11e0 2024-09-18T01:07:44.389 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:44.384+0000 7f849c00d640 -1 reset not still connected to 0x5629e4d35a00 2024-09-18T01:07:44.389 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:44.384+0000 7f849c00d640 -1 reset not still connected to 0x5629f09b1040 2024-09-18T01:07:44.389 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:44.384+0000 7f849c00d640 -1 reset not still connected to 0x5629f09b1110 2024-09-18T01:07:44.389 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:44.384+0000 7f849c00d640 -1 reset not still connected to 0x5629f09b11e0 2024-09-18T01:07:45.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:07:45.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary2 2024-09-18T01:07:45.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:45.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:45.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:07:45.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:45.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:07:45.587 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary2 2024-09-18T01:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary2 2024-09-18T01:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T01:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary3 2024-09-18T01:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+replaying 2024-09-18T01:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:45.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:07:45.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:45.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:45.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:45.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:45.630 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:07:46.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+replaying '' '' 2024-09-18T01:07:46.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:07:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:46.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:07:46.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:07:46.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T01:07:46.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T01:07:46.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 4a901bdb-c18f-43d9-a0f0-efe4d94ef3aa 2024-09-18T01:07:46.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:07:46.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64069632.0,"last_snapshot_bytes":64069632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"} 2024-09-18T01:07:46.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:07:46.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:39 2024-09-18T01:07:46.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:07:46.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:07:46.724 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:07:46.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:07:46.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:44' 2024-09-18T01:07:46.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:07:46.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:07:46.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:07:46.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64069632.0,"last_snapshot_bytes":64069632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"} 2024-09-18T01:07:46.727 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:07:46.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:07:46.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:07:46.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:07:46.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:07:46.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:07:46.731 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:07:46.732 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:07:46.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:07:46.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+replaying '2024-09-18 01:07:39' 2024-09-18T01:07:46.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:46.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:46.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:07:46.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:46.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:07:39' 2024-09-18T01:07:46.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:07:46.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:07:46.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:07:47.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK842544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10943aeb36-cdea-4614-a073-08041a451b48up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70389760.0,"last_snapshot_bytes":70389760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:44image-primary34a901bdb-c18f-43d9-a0f0-efe4d94ef3aaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":64069632.0,"last_snapshot_bytes":64069632,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:44image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621545,"remote_snapshot_timestamp":1726621545,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:44image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":133029888.0,"last_snapshot_bytes":133029888,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621544,"remote_snapshot_timestamp":1726621544,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:44image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70172672.0,"last_snapshot_bytes":70172672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:44image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70217728.0,"last_snapshot_bytes":70217728,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:44image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70197248.0,"last_snapshot_bytes":70197248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:44image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70287360.0,"last_snapshot_bytes":70287360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:44' 2024-09-18T01:07:47.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:07:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T01:07:47.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:07:47.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T01:07:47.125 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:07:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:07:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:07:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:07:47.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:47.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:47.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary3 2024-09-18T01:07:47.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:07:47.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:47.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:07:47.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary3 2024-09-18T01:07:47.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary3 2024-09-18T01:07:48.105 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:48.096+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T01:07:48.105 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:48.096+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T01:07:48.105 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:48.096+0000 7f849c00d640 -1 reset not still connected to 0x5629e21a3790 2024-09-18T01:07:48.105 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:48.096+0000 7f849c00d640 -1 reset not still connected to 0x5629e4d35a00 2024-09-18T01:07:48.105 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:48.096+0000 7f849c00d640 -1 reset not still connected to 0x5629f09b1040 2024-09-18T01:07:48.106 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:07:48.100+0000 7f849c00d640 -1 reset not still connected to 0x5629f09b1110 2024-09-18T01:07:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:07:49.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary3 2024-09-18T01:07:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:07:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:07:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary3 2024-09-18T01:07:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:49.614 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary3 2024-09-18T01:07:49.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:07:49.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T01:07:49.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary4 2024-09-18T01:07:49.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:49.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:49.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:07:49.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:07:49.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+replaying 2024-09-18T01:07:49.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:49.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:49.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:07:49.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:49.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:49.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:49.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:49.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:07:50.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+replaying '' '' 2024-09-18T01:07:50.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:50.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:50.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:07:50.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:50.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:50.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:50.617 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:07:50.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:07:50.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T01:07:50.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T01:07:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f16bbff5-762b-414e-93aa-bcfe90e66fb3 2024-09-18T01:07:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:07:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621545,"remote_snapshot_timestamp":1726621545,"replay_state":"idle"} 2024-09-18T01:07:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:07:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:39 2024-09-18T01:07:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:07:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:07:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:07:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:07:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:49' 2024-09-18T01:07:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:07:50.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:07:50.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:07:50.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621545,"remote_snapshot_timestamp":1726621545,"replay_state":"idle"} 2024-09-18T01:07:50.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:07:50.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:07:50.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:07:50.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:07:50.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:07:50.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:07:50.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:07:50.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:07:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:07:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+replaying '2024-09-18 01:07:39' 2024-09-18T01:07:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:07:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:07:39' 2024-09-18T01:07:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:07:50.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:07:50.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:07:50.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10943aeb36-cdea-4614-a073-08041a451b48up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70389760.0,"last_snapshot_bytes":70389760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:49image-primary4f16bbff5-762b-414e-93aa-bcfe90e66fb3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70176768.0,"last_snapshot_bytes":70176768,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621545,"remote_snapshot_timestamp":1726621545,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:49image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":133029888.0,"last_snapshot_bytes":133029888,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621544,"remote_snapshot_timestamp":1726621544,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:49image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70172672.0,"last_snapshot_bytes":70172672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:49image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70217728.0,"last_snapshot_bytes":70217728,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:49image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70197248.0,"last_snapshot_bytes":70197248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:49image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70287360.0,"last_snapshot_bytes":70287360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:49' 2024-09-18T01:07:50.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:07:50.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T01:07:50.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:07:50.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T01:07:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:07:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:07:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:07:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:07:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary4 2024-09-18T01:07:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:07:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:07:50.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary4 2024-09-18T01:07:50.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary4 2024-09-18T01:07:53.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:07:53.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary4 2024-09-18T01:07:53.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:53.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:53.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:07:53.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:53.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:07:53.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary4 2024-09-18T01:07:53.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:53.126 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary4 2024-09-18T01:07:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:07:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T01:07:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary5 2024-09-18T01:07:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:07:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:07:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+replaying 2024-09-18T01:07:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:07:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:53.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:53.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:53.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:53.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:07:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+replaying '' '' 2024-09-18T01:07:54.128 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:07:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:54.129 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:54.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:07:54.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:07:54.130 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T01:07:54.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T01:07:54.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 5984c013-9b62-4bf1-bc42-f8038f39f85c 2024-09-18T01:07:54.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:07:54.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":133029888.0,"last_snapshot_bytes":133029888,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621544,"remote_snapshot_timestamp":1726621544,"replay_state":"idle"} 2024-09-18T01:07:54.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:07:54.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:39 2024-09-18T01:07:54.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:07:54.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:07:54.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:07:54.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:07:54.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:49' 2024-09-18T01:07:54.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:07:54.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:07:54.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:07:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":133029888.0,"last_snapshot_bytes":133029888,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621544,"remote_snapshot_timestamp":1726621544,"replay_state":"idle"} 2024-09-18T01:07:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:07:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:07:54.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:07:54.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:07:54.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:07:54.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:07:54.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:07:54.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:07:54.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+replaying '2024-09-18 01:07:39' 2024-09-18T01:07:54.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:54.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:54.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:07:54.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:54.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:07:39' 2024-09-18T01:07:54.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:07:54.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:07:54.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:07:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK642544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10943aeb36-cdea-4614-a073-08041a451b48up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70389760.0,"last_snapshot_bytes":70389760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:54image-primary55984c013-9b62-4bf1-bc42-f8038f39f85cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":133029888.0,"last_snapshot_bytes":133029888,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621544,"remote_snapshot_timestamp":1726621544,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:54image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70172672.0,"last_snapshot_bytes":70172672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:54image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70217728.0,"last_snapshot_bytes":70217728,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:54image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70197248.0,"last_snapshot_bytes":70197248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:54image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70287360.0,"last_snapshot_bytes":70287360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:54' 2024-09-18T01:07:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:07:54.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T01:07:54.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:07:54.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T01:07:54.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:07:54.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:07:54.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:07:54.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:07:54.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:54.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:54.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary5 2024-09-18T01:07:54.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:07:54.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:54.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:07:54.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary5 2024-09-18T01:07:54.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary5 2024-09-18T01:07:56.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:07:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary5 2024-09-18T01:07:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:07:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:07:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary5 2024-09-18T01:07:56.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:56.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary5 2024-09-18T01:07:56.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:07:56.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T01:07:56.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary6 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+replaying 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:56.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:07:57.121 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+replaying '' '' 2024-09-18T01:07:57.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:57.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:57.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:07:57.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:57.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:57.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:57.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:07:57.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:07:57.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T01:07:57.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T01:07:57.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 97f2fe38-61d0-453f-b29c-085ba52e905a 2024-09-18T01:07:57.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:07:57.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70172672.0,"last_snapshot_bytes":70172672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"} 2024-09-18T01:07:57.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:07:57.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:39 2024-09-18T01:07:57.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:07:57.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:07:57.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:07:57.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:07:57.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:54' 2024-09-18T01:07:57.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:07:57.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:07:57.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:07:57.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70172672.0,"last_snapshot_bytes":70172672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"} 2024-09-18T01:07:57.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:07:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:07:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:07:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:07:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:07:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:07:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:07:57.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:07:57.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:07:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+replaying '2024-09-18 01:07:39' 2024-09-18T01:07:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:07:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:07:39' 2024-09-18T01:07:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:07:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:07:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:07:57.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10943aeb36-cdea-4614-a073-08041a451b48up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70389760.0,"last_snapshot_bytes":70389760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:54image-primary697f2fe38-61d0-453f-b29c-085ba52e905aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70172672.0,"last_snapshot_bytes":70172672,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:54image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70217728.0,"last_snapshot_bytes":70217728,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:54image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70197248.0,"last_snapshot_bytes":70197248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:54image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70287360.0,"last_snapshot_bytes":70287360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:54' 2024-09-18T01:07:57.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:07:57.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T01:07:57.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:07:57.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T01:07:57.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:07:57.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:07:57.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:07:57.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:07:57.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:57.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:57.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary6 2024-09-18T01:07:57.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:07:57.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:57.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:07:57.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary6 2024-09-18T01:07:57.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary6 2024-09-18T01:07:59.111 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:07:59.115 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary6 2024-09-18T01:07:59.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:59.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:59.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:07:59.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:59.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:07:59.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary6 2024-09-18T01:07:59.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:07:59.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary6 2024-09-18T01:07:59.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:07:59.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T01:07:59.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary7 2024-09-18T01:07:59.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:59.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:59.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:07:59.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:07:59.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+replaying 2024-09-18T01:07:59.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:07:59.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:07:59.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:07:59.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:59.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:07:59.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:07:59.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:59.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:08:00.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+replaying '' '' 2024-09-18T01:08:00.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:08:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:08:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:08:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:08:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:08:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T01:08:00.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T01:08:00.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 225f2829-01d3-45e2-a1d6-0f790148e4c3 2024-09-18T01:08:00.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:08:00.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70217728.0,"last_snapshot_bytes":70217728,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"} 2024-09-18T01:08:00.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:08:00.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:39 2024-09-18T01:08:00.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:08:00.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:08:00.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:08:00.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:08:00.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:59' 2024-09-18T01:08:00.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:08:00.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:08:00.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:08:00.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70217728.0,"last_snapshot_bytes":70217728,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"} 2024-09-18T01:08:00.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:08:00.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:08:00.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:08:00.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:08:00.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:08:00.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:08:00.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:08:00.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:08:00.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:08:00.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+replaying '2024-09-18 01:07:39' 2024-09-18T01:08:00.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:00.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:00.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:00.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:08:00.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:07:39' 2024-09-18T01:08:00.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:08:00.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:08:00.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:08:00.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10943aeb36-cdea-4614-a073-08041a451b48up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70389760.0,"last_snapshot_bytes":70389760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:59image-primary7225f2829-01d3-45e2-a1d6-0f790148e4c3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70217728.0,"last_snapshot_bytes":70217728,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:59image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70197248.0,"last_snapshot_bytes":70197248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:59image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70287360.0,"last_snapshot_bytes":70287360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:59' 2024-09-18T01:08:00.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:08:00.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T01:08:00.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:08:00.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T01:08:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:08:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:08:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:08:00.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:08:00.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:00.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:00.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary7 2024-09-18T01:08:00.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:08:00.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:00.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:00.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary7 2024-09-18T01:08:00.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary7 2024-09-18T01:08:00.568 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:00.564+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T01:08:00.569 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:00.564+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T01:08:00.569 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:00.564+0000 7f849c00d640 -1 reset not still connected to 0x5629d8f35110 2024-09-18T01:08:00.569 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:00.564+0000 7f849c00d640 -1 reset not still connected to 0x5629f09b1040 2024-09-18T01:08:02.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:08:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary7 2024-09-18T01:08:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:08:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:08:02.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary7 2024-09-18T01:08:02.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:02.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary7 2024-09-18T01:08:02.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:02.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T01:08:02.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary8 2024-09-18T01:08:02.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:02.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:02.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:02.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:08:02.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+replaying 2024-09-18T01:08:02.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:02.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:02.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:02.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:08:02.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:08:02.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:08:02.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:08:02.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:08:03.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+replaying '' '' 2024-09-18T01:08:03.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:03.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:03.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:03.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:08:03.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:08:03.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:08:03.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:08:03.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:08:03.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T01:08:03.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T01:08:03.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: f1cbc639-19c2-4066-bbe2-5861c8b3f55a 2024-09-18T01:08:03.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:08:03.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70197248.0,"last_snapshot_bytes":70197248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"} 2024-09-18T01:08:03.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:08:03.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:39 2024-09-18T01:08:03.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:08:03.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:08:03.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:08:03.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:08:03.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:59' 2024-09-18T01:08:03.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:08:03.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:08:03.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:08:03.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70197248.0,"last_snapshot_bytes":70197248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"} 2024-09-18T01:08:03.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:08:03.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:08:03.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:08:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:08:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:08:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:08:03.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:08:03.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:08:03.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:08:03.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+replaying '2024-09-18 01:07:39' 2024-09-18T01:08:03.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:03.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:03.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:03.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:08:03.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:07:39' 2024-09-18T01:08:03.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:08:03.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:08:03.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:08:03.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK342544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10943aeb36-cdea-4614-a073-08041a451b48up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70389760.0,"last_snapshot_bytes":70389760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:59image-primary8f1cbc639-19c2-4066-bbe2-5861c8b3f55aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70197248.0,"last_snapshot_bytes":70197248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:59image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70287360.0,"last_snapshot_bytes":70287360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:07:59' 2024-09-18T01:08:03.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:08:03.445 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T01:08:03.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:08:03.451 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T01:08:03.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:08:03.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:08:03.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:08:03.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:08:03.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:03.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:03.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary8 2024-09-18T01:08:03.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:08:03.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:03.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:03.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary8 2024-09-18T01:08:03.516 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary8 2024-09-18T01:08:05.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:08:05.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary8 2024-09-18T01:08:05.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:05.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:05.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:05.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:08:05.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:08:05.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary8 2024-09-18T01:08:05.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:05.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary8 2024-09-18T01:08:05.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:05.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T01:08:05.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary9 2024-09-18T01:08:05.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:05.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:05.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:05.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:08:05.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+replaying 2024-09-18T01:08:05.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:05.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:08:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:08:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:08:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:08:05.274 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:08:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+replaying '' '' 2024-09-18T01:08:06.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:06.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:06.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:06.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:08:06.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:08:06.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:08:06.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:08:06.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:08:06.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T01:08:06.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T01:08:06.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 36b71294-3a4d-45ce-bfdd-d89c2dfd7337 2024-09-18T01:08:06.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:08:06.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70287360.0,"last_snapshot_bytes":70287360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"} 2024-09-18T01:08:06.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:08:06.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:07:39 2024-09-18T01:08:06.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:08:06.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:08:06.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:08:06.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:08:06.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:08:04' 2024-09-18T01:08:06.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:08:06.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:08:06.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:08:06.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70287360.0,"last_snapshot_bytes":70287360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"} 2024-09-18T01:08:06.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:08:06.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:08:06.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:08:06.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:08:06.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:08:06.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:08:06.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:08:06.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:08:06.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:08:06.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+replaying '2024-09-18 01:07:39' 2024-09-18T01:08:06.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:06.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:06.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:06.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:08:06.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:07:39' 2024-09-18T01:08:06.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:08:06.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:08:06.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:08:06.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10943aeb36-cdea-4614-a073-08041a451b48up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70389760.0,"last_snapshot_bytes":70389760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:08:04image-primary936b71294-3a4d-45ce-bfdd-d89c2dfd7337up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70287360.0,"last_snapshot_bytes":70287360,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621542,"remote_snapshot_timestamp":1726621542,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:07:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:08:04' 2024-09-18T01:08:06.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:08:06.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T01:08:06.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:07:39' 2024-09-18T01:08:06.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T01:08:06.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:08:06.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:08:06.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:08:06.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:08:06.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:06.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:06.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary9 2024-09-18T01:08:06.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:08:06.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:06.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:06.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary9 2024-09-18T01:08:06.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary9 2024-09-18T01:08:08.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:08:08.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary9 2024-09-18T01:08:08.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:08.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:08.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:08.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:08:08.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:08:08.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary9 2024-09-18T01:08:08.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:08.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary9 2024-09-18T01:08:08.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:08.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T01:08:08.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary10 2024-09-18T01:08:08.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:08.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:08.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:08.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:08:08.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+replaying 2024-09-18T01:08:08.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:08.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:08.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:08.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:08:08.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:08:08.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:08:08.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:08:08.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:08:08.661 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T01:08:08.655+0000 7fac32fac640 -1 reset not still connected to 0x55a839605ba0 2024-09-18T01:08:08.661 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T01:08:08.655+0000 7fac32fac640 -1 reset not still connected to 0x55a83a5192b0 2024-09-18T01:08:08.661 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T01:08:08.655+0000 7fac32fac640 -1 reset not still connected to 0x55a83a519e10 2024-09-18T01:08:08.661 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T01:08:08.655+0000 7fac32fac640 -1 reset not still connected to 0x55a8658891e0 2024-09-18T01:08:09.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+replaying '' '' 2024-09-18T01:08:09.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:09.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:09.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:09.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:08:09.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:08:09.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:08:09.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:08:09.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:08:09.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T01:08:09.393 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T01:08:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 943aeb36-cdea-4614-a073-08041a451b48 2024-09-18T01:08:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:08:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70389760.0,"last_snapshot_bytes":70389760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"} 2024-09-18T01:08:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:08:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:08:09 2024-09-18T01:08:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:08:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:08:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:08:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:08:09.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:08:09' 2024-09-18T01:08:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:08:09.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:08:09.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:08:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70389760.0,"last_snapshot_bytes":70389760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"} 2024-09-18T01:08:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:08:09.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:08:09.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:08:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:08:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:08:09.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:08:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:08:09.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:08:09.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:08:09' 2024-09-18T01:08:09.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+replaying '2024-09-18 01:08:09' 2024-09-18T01:08:09.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:09.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:09.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:09.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:08:09.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:08:09' 2024-09-18T01:08:09.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:08:09.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:08:09.408 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:08:09.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary10943aeb36-cdea-4614-a073-08041a451b48up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70389760.0,"last_snapshot_bytes":70389760,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621543,"remote_snapshot_timestamp":1726621543,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:08:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:08:09' 2024-09-18T01:08:09.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:08:09.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T01:08:09.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:08:09' 2024-09-18T01:08:09.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T01:08:09.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:08:09.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:08:09.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:08:09.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:08:09.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:09.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:09.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary10 2024-09-18T01:08:09.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:08:09.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:09.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:09.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary10 2024-09-18T01:08:09.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary10 2024-09-18T01:08:11.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:08:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary10 2024-09-18T01:08:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:08:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:08:11.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary10 2024-09-18T01:08:11.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:11.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary10 2024-09-18T01:08:11.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..10} 2024-09-18T01:08:11.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS=() 2024-09-18T01:08:11.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS=() 2024-09-18T01:08:11.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS=() 2024-09-18T01:08:11.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS=() 2024-09-18T01:08:11.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ RET=0 2024-09-18T01:08:11.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:11.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T01:08:11.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary1 2024-09-18T01:08:11.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary1 snapshot 10G 2024-09-18T01:08:11.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:11.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:11.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:11.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:11.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:11.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:11.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:11.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T01:08:11.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary1 10G 2024-09-18T01:08:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T01:08:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T01:08:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T01:08:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:11.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary1 2024-09-18T01:08:11.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:08:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T01:08:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary1 snapshot 2024-09-18T01:08:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary1 snapshot 2024-09-18T01:08:11.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T01:08:11.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary1 2024-09-18T01:08:12.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary1': 2024-09-18T01:08:12.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T01:08:12.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T01:08:12.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T01:08:12.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 3966747884d9 2024-09-18T01:08:12.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.3966747884d9 2024-09-18T01:08:12.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T01:08:12.050 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T01:08:12.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T01:08:12.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T01:08:12.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 01:08:11 2024 2024-09-18T01:08:12.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 01:08:11 2024 2024-09-18T01:08:12.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 01:08:11 2024 2024-09-18T01:08:12.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T01:08:12.051 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T01:08:12.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: d0f03b0b-70ae-444b-9369-278125560577 2024-09-18T01:08:12.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T01:08:12.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:08:12.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T01:08:12.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd0 2024-09-18T01:08:12.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T01:08:12.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd0 2024-09-18T01:08:12.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T01:08:12.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T01:08:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T01:08:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 4ac62199-b46f-47ba-a5c0-f79a6fc05d77 2024-09-18T01:08:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T01:08:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T01:08:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T01:08:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T01:08:12.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T01:08:12.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T01:08:12.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary1 2024-09-18T01:08:12.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd0 test-primary1 2024-09-18T01:08:12.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T01:08:12.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary1 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary1 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary1 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary2 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary2 snapshot 10G 2024-09-18T01:08:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:08:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:08:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary1 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary2 10G 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary1 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T01:08:12.864 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T01:08:12.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:12.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary2 2024-09-18T01:08:12.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:08:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T01:08:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary2 snapshot 2024-09-18T01:08:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary2 snapshot 2024-09-18T01:08:13.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2977 2024-09-18T01:08:13.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T01:08:13.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:13.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary2 2024-09-18T01:08:13.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary2': 2024-09-18T01:08:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T01:08:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T01:08:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T01:08:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 3975a4c4a8d8 2024-09-18T01:08:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.3975a4c4a8d8 2024-09-18T01:08:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T01:08:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T01:08:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T01:08:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T01:08:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 01:08:12 2024 2024-09-18T01:08:13.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 01:08:12 2024 2024-09-18T01:08:13.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 01:08:12 2024 2024-09-18T01:08:13.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T01:08:13.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T01:08:13.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 68158950-cc3b-4cfd-bf00-d7829fce7865 2024-09-18T01:08:13.284 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T01:08:13.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:08:13.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T01:08:13.375 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:08:13.371+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef1c70 2024-09-18T01:08:13.375 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:08:13.371+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4000 2024-09-18T01:08:13.376 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:08:13.371+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4750 2024-09-18T01:08:13.376 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:08:13.371+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6734b1ee0 2024-09-18T01:08:13.376 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:08:13.371+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735afee0 2024-09-18T01:08:13.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd1 2024-09-18T01:08:13.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T01:08:13.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd1 2024-09-18T01:08:13.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T01:08:13.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T01:08:13.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T01:08:13.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 856bdeb1-d790-425f-90a4-fdae4fe37f03 2024-09-18T01:08:13.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T01:08:13.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T01:08:13.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:13.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T01:08:13.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T01:08:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T01:08:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary1/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary1' 2024-09-18T01:08:13.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T01:08:14.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T01:08:14.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:14.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary2 2024-09-18T01:08:14.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd1 test-primary2 2024-09-18T01:08:14.145 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T01:08:14.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary2 2024-09-18T01:08:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T01:08:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary2 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary3 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary2 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary3 snapshot 10G 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary2 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary3 10G 2024-09-18T01:08:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:08:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary2 2024-09-18T01:08:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T01:08:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T01:08:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T01:08:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary3 2024-09-18T01:08:14.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:08:14.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T01:08:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary3 snapshot 2024-09-18T01:08:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary3 snapshot 2024-09-18T01:08:14.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T01:08:14.281 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary2/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary2' 2024-09-18T01:08:15.150 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2979 2024-09-18T01:08:15.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:15.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T01:08:15.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary3 2024-09-18T01:08:15.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary3': 2024-09-18T01:08:15.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T01:08:15.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T01:08:15.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T01:08:15.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 39866b4522ae 2024-09-18T01:08:15.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.39866b4522ae 2024-09-18T01:08:15.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T01:08:15.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T01:08:15.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T01:08:15.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T01:08:15.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 01:08:14 2024 2024-09-18T01:08:15.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 01:08:14 2024 2024-09-18T01:08:15.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 01:08:14 2024 2024-09-18T01:08:15.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T01:08:15.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T01:08:15.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 821edf46-8d24-4ea2-bb84-66b4b58ea23a 2024-09-18T01:08:15.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T01:08:15.328 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:08:15.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T01:08:15.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd2 2024-09-18T01:08:15.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T01:08:15.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd2 2024-09-18T01:08:15.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T01:08:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T01:08:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T01:08:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 34a3619c-6e5a-44ba-aee2-4f2f32be572a 2024-09-18T01:08:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T01:08:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T01:08:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T01:08:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T01:08:15.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T01:08:15.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T01:08:15.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:15.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary3 2024-09-18T01:08:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd2 test-primary3 2024-09-18T01:08:16.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T01:08:16.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary3 2024-09-18T01:08:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T01:08:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary3 2024-09-18T01:08:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T01:08:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:16.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T01:08:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:08:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary4 2024-09-18T01:08:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary4 snapshot 10G 2024-09-18T01:08:16.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:16.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:16.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary3 2024-09-18T01:08:16.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:08:16.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:16.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:16.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:16.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T01:08:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary4 10G 2024-09-18T01:08:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary3 2024-09-18T01:08:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T01:08:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary3 2024-09-18T01:08:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T01:08:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T01:08:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary4 2024-09-18T01:08:16.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:08:16.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:16.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T01:08:16.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:16.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary4 snapshot 2024-09-18T01:08:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:16.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary4 snapshot 2024-09-18T01:08:16.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T01:08:16.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary3/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary3' 2024-09-18T01:08:16.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:16.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:08:16.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:16.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:16.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:16.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:08:17.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2981 2024-09-18T01:08:17.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:17.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T01:08:17.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary4 2024-09-18T01:08:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary4': 2024-09-18T01:08:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T01:08:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T01:08:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T01:08:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 3995e07bbdad 2024-09-18T01:08:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.3995e07bbdad 2024-09-18T01:08:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T01:08:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T01:08:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T01:08:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T01:08:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 01:08:16 2024 2024-09-18T01:08:17.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 01:08:16 2024 2024-09-18T01:08:17.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 01:08:16 2024 2024-09-18T01:08:17.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T01:08:17.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T01:08:17.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: ee4c71cc-a845-47e1-b515-dffaef47a382 2024-09-18T01:08:17.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T01:08:17.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:08:17.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T01:08:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd3 2024-09-18T01:08:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T01:08:17.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd3 2024-09-18T01:08:17.429 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T01:08:17.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T01:08:17.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T01:08:17.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 6517a00a-0584-4829-bf48-a7e6e7c993e7 2024-09-18T01:08:17.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T01:08:17.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T01:08:17.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:17.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T01:08:17.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T01:08:17.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T01:08:17.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T01:08:17.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:17.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary4 2024-09-18T01:08:17.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd3 test-primary4 2024-09-18T01:08:17.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T01:08:17.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary4 2024-09-18T01:08:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T01:08:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary4 2024-09-18T01:08:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:08:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:08:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T01:08:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary4 2024-09-18T01:08:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T01:08:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary4 2024-09-18T01:08:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary5 2024-09-18T01:08:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary4 2024-09-18T01:08:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary5 snapshot 10G 2024-09-18T01:08:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T01:08:17.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:17.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary5 10G 2024-09-18T01:08:17.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T01:08:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T01:08:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T01:08:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:17.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary5 2024-09-18T01:08:18.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2983 2024-09-18T01:08:18.142 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T01:08:18.143 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary4/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary4' 2024-09-18T01:08:18.148 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:08:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:08:18.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:08:18.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:18.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T01:08:18.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:18.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary5 snapshot 2024-09-18T01:08:18.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:18.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:18.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:18.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:18.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary5 snapshot 2024-09-18T01:08:19.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2984 2024-09-18T01:08:19.238 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:19.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T01:08:19.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary5 2024-09-18T01:08:19.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary5': 2024-09-18T01:08:19.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 39a9bde382a9 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.39a9bde382a9 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 01:08:18 2024 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 01:08:18 2024 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 01:08:18 2024 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T01:08:19.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T01:08:19.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 90242e82-997f-4088-b460-92f5b78a8509 2024-09-18T01:08:19.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T01:08:19.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:08:19.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T01:08:19.392 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:19.383+0000 7f849c00d640 -1 reset not still connected to 0x5629a85e6f70 2024-09-18T01:08:19.392 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:19.383+0000 7f849c00d640 -1 reset not still connected to 0x5629a85e7040 2024-09-18T01:08:19.392 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:19.383+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T01:08:19.392 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:19.383+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T01:08:19.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd4 2024-09-18T01:08:19.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T01:08:19.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd4 2024-09-18T01:08:19.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T01:08:19.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T01:08:19.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T01:08:19.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 711c3dc9-ec28-4c52-afd4-7f7227d074f8 2024-09-18T01:08:19.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T01:08:19.625 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T01:08:19.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:19.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T01:08:19.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T01:08:19.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T01:08:20.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 Snapshot ID: 2985 2024-09-18T01:08:20.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T01:08:20.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary5 2024-09-18T01:08:20.177 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd4 test-primary5 2024-09-18T01:08:20.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:08:20.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:20.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:08:20.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T01:08:20.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary5 2024-09-18T01:08:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T01:08:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary5 2024-09-18T01:08:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T01:08:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary5 2024-09-18T01:08:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T01:08:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary6 2024-09-18T01:08:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary6 snapshot 10G 2024-09-18T01:08:20.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary5 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T01:08:20.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary5 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary6 10G 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T01:08:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary6 2024-09-18T01:08:20.376 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:08:20.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:20.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T01:08:20.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:20.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary6 snapshot 2024-09-18T01:08:20.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:20.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:20.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:08:20.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:20.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary6 snapshot 2024-09-18T01:08:20.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T01:08:20.381 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary5/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary5' 2024-09-18T01:08:21.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:21.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:08:21.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:21.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:21.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:21.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:08:21.160 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2988 2024-09-18T01:08:21.165 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T01:08:21.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:21.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary6 2024-09-18T01:08:21.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary6': 2024-09-18T01:08:21.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T01:08:21.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T01:08:21.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T01:08:21.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 39bceb670481 2024-09-18T01:08:21.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.39bceb670481 2024-09-18T01:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T01:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T01:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T01:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T01:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 01:08:20 2024 2024-09-18T01:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 01:08:20 2024 2024-09-18T01:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 01:08:20 2024 2024-09-18T01:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T01:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T01:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 977eb8ba-b7cf-46d7-bbaf-d2f345b9d999 2024-09-18T01:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T01:08:21.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:08:21.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T01:08:21.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd5 2024-09-18T01:08:21.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T01:08:21.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd5 2024-09-18T01:08:21.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T01:08:21.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T01:08:21.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T01:08:21.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 45ce9b4b-8220-4864-885e-ccb1815adcd3 2024-09-18T01:08:21.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T01:08:21.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T01:08:21.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:21.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T01:08:21.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T01:08:21.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T01:08:21.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T01:08:21.999 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:22.001 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary6 2024-09-18T01:08:22.003 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd5 test-primary6 2024-09-18T01:08:22.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T01:08:22.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary6 2024-09-18T01:08:22.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T01:08:22.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary6 2024-09-18T01:08:22.095 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:22.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T01:08:22.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:22.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:22.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:08:22.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T01:08:22.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary7 2024-09-18T01:08:22.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:22.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary7 snapshot 10G 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary7 10G 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:22.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary7 2024-09-18T01:08:22.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary6 2024-09-18T01:08:22.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary6 2024-09-18T01:08:22.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary6 2024-09-18T01:08:22.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2987 2024-09-18T01:08:22.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:22.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:08:22.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:22.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T01:08:22.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:22.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary7 snapshot 2024-09-18T01:08:22.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:22.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:22.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:22.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:22.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary7 snapshot 2024-09-18T01:08:22.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T01:08:22.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary6/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary6' 2024-09-18T01:08:22.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:22.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:08:22.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:22.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:22.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:22.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:08:23.140 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2990 2024-09-18T01:08:23.172 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:08:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:08:23.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T01:08:23.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2991 2024-09-18T01:08:23.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:23.314 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary7 2024-09-18T01:08:23.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary7': 2024-09-18T01:08:23.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T01:08:23.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 39ce397ffd6a 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.39ce397ffd6a 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 01:08:22 2024 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 01:08:22 2024 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 01:08:22 2024 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T01:08:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 9e5e582f-3b8d-48a4-8031-9e27f633478c 2024-09-18T01:08:23.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T01:08:23.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:08:23.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T01:08:23.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd6 2024-09-18T01:08:23.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T01:08:23.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd6 2024-09-18T01:08:23.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T01:08:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T01:08:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T01:08:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: 7e9255a2-0c6d-4773-8c13-22a9c04f7fc7 2024-09-18T01:08:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T01:08:23.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T01:08:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T01:08:23.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T01:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T01:08:24.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T01:08:24.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:24.100 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary7 2024-09-18T01:08:24.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd6 test-primary7 2024-09-18T01:08:24.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:24.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:08:24.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:24.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:24.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:24.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:08:24.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2993 2024-09-18T01:08:24.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:24.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T01:08:24.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary7 2024-09-18T01:08:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T01:08:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T01:08:24.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary7 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary7 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary8 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary8 snapshot 10G 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary7 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:24.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary7 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary8 10G 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:24.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:24.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T01:08:24.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T01:08:24.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T01:08:24.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:08:24.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:24.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:24.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary8 2024-09-18T01:08:24.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:24.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:24.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:08:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T01:08:24.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary7/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary7' 2024-09-18T01:08:24.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:08:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T01:08:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary8 snapshot 2024-09-18T01:08:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:24.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary8 snapshot 2024-09-18T01:08:25.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2994 2024-09-18T01:08:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:08:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:25.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:08:25.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:25.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T01:08:25.268 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary8 2024-09-18T01:08:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2996 2024-09-18T01:08:25.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary8': 2024-09-18T01:08:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T01:08:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T01:08:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T01:08:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 39e42cfb6302 2024-09-18T01:08:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.39e42cfb6302 2024-09-18T01:08:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T01:08:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T01:08:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T01:08:25.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T01:08:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 01:08:24 2024 2024-09-18T01:08:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 01:08:24 2024 2024-09-18T01:08:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 01:08:24 2024 2024-09-18T01:08:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T01:08:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T01:08:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 8c413c2e-0975-4386-8c15-1b1c7c115008 2024-09-18T01:08:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T01:08:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:08:25.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T01:08:25.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd7 2024-09-18T01:08:25.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T01:08:25.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd7 2024-09-18T01:08:25.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T01:08:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T01:08:25.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T01:08:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: eb64a6a9-a7d4-4f3f-a43e-6ed9cc8f1a3a 2024-09-18T01:08:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T01:08:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T01:08:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:25.633 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T01:08:25.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T01:08:26.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T01:08:26.173 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:26.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:08:26.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:26.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:26.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:26.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:08:26.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 Snapshot ID: 2995 2024-09-18T01:08:26.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:done 2024-09-18T01:08:26.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:26.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary8 2024-09-18T01:08:26.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:26.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd7 test-primary8 2024-09-18T01:08:26.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:26.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:08:26.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:26.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:26.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:08:26.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:08:26.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T01:08:26.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary8 2024-09-18T01:08:26.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T01:08:26.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary8 2024-09-18T01:08:26.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:26.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T01:08:26.481 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T01:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary9 2024-09-18T01:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary8 2024-09-18T01:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:08:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary9 snapshot 10G 2024-09-18T01:08:26.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:26.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:26.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:26.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:26.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:26.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:26.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:26.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T01:08:26.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:26.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary9 10G 2024-09-18T01:08:26.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary8 2024-09-18T01:08:26.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:26.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:26.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary8 2024-09-18T01:08:26.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:26.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:26.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:26.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:26.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T01:08:26.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T01:08:26.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T01:08:26.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:26.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary9 2024-09-18T01:08:26.605 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T01:08:26.606 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary8/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary8' 2024-09-18T01:08:26.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:08:26.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:26.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T01:08:26.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:26.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary9 snapshot 2024-09-18T01:08:26.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:26.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:26.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:26.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:26.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary9 snapshot 2024-09-18T01:08:27.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2998 2024-09-18T01:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:08:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:27.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:08:27.236 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T01:08:27.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary9 2024-09-18T01:08:27.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3001 2024-09-18T01:08:27.335 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary9': 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 39ff62b1ad92 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.39ff62b1ad92 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 01:08:26 2024 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 01:08:26 2024 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 01:08:26 2024 2024-09-18T01:08:27.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T01:08:27.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T01:08:27.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: b5da1212-bf20-4dc9-bf59-f911a78837d4 2024-09-18T01:08:27.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T01:08:27.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:08:27.340 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T01:08:27.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:27.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd8 2024-09-18T01:08:27.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T01:08:27.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd8 2024-09-18T01:08:27.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T01:08:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T01:08:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T01:08:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: ec7d8c99-f7f0-4cdb-850e-78148b1a386f 2024-09-18T01:08:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T01:08:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T01:08:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T01:08:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T01:08:27.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T01:08:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T01:08:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:28.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary9 2024-09-18T01:08:28.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd8 test-primary9 2024-09-18T01:08:28.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3000 2024-09-18T01:08:28.152 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:28.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 2999 2024-09-18T01:08:28.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:28.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:28.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:08:28.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:28.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:28.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:28.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:08:28.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T01:08:28.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary9 2024-09-18T01:08:28.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T01:08:28.259 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ MNTPT=test-primary10 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary9 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image_and_enable_mirror cluster1 mirror image-primary10 snapshot 10G 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n snapshot ']' 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary9 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ create_image cluster1 mirror image-primary10 10G 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary9 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local size=128 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 10G ']' 2024-09-18T01:08:28.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ size=10G 2024-09-18T01:08:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ shift 2024-09-18T01:08:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image-primary10 2024-09-18T01:08:28.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary9 2024-09-18T01:08:28.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:28.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:28.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:28.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:28.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:08:28.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:28.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:28.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:28.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:08:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:08:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:08:28.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:08:28.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:28.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' image = image ']' 2024-09-18T01:08:28.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:08:28.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ enable_mirror cluster1 mirror image-primary10 snapshot 2024-09-18T01:08:28.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:28.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:28.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:28.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mode=snapshot 2024-09-18T01:08:28.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image-primary10 snapshot 2024-09-18T01:08:28.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T01:08:28.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary9/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary9' 2024-09-18T01:08:29.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:29.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:08:29.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:29.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:29.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:29.221 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:08:29.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3003 2024-09-18T01:08:29.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3006 2024-09-18T01:08:29.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:29.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:29.313 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Mirroring enabled 2024-09-18T01:08:29.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 info mirror/image-primary10 2024-09-18T01:08:29.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:rbd image 'image-primary10': 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: size 10 GiB in 2560 objects 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: order 22 (4 MiB objects) 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: snapshot_count: 1 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: id: 3a167bed9622 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: block_name_prefix: rbd_data.3a167bed9622 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: format: 2 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: op_features: 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: flags: 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: create_timestamp: Wed Sep 18 01:08:28 2024 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: access_timestamp: Wed Sep 18 01:08:28 2024 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: modify_timestamp: Wed Sep 18 01:08:28 2024 2024-09-18T01:08:29.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring state: enabled 2024-09-18T01:08:29.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring mode: snapshot 2024-09-18T01:08:29.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring global id: 01a5a6a4-a190-4d6c-9392-bbfa30f6ec37 2024-09-18T01:08:29.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: mirroring primary: true 2024-09-18T01:08:29.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:08:29.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster1 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T01:08:29.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEV=/dev/nbd9 2024-09-18T01:08:29.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ DEVS+=($DEV) 2024-09-18T01:08:29.561 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mkfs.ext4 /dev/nbd9 2024-09-18T01:08:29.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:mke2fs 1.46.5 (30-Dec-2021) 2024-09-18T01:08:29.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Discarding device blocks: 0/2621440 done 2024-09-18T01:08:29.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating filesystem with 2621440 4k blocks and 655360 inodes 2024-09-18T01:08:29.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Filesystem UUID: ccb2593a-4cab-4f8d-ace0-e0e2ee9a20cc 2024-09-18T01:08:29.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Superblock backups stored on blocks: 2024-09-18T01:08:29.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632 2024-09-18T01:08:29.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:29.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Allocating group tables: 0/80 done 2024-09-18T01:08:29.624 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing inode tables: 0/80 done 2024-09-18T01:08:29.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Creating journal (16384 blocks): done 2024-09-18T01:08:30.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Writing superblocks and filesystem accounting information: 0/80 done 2024-09-18T01:08:30.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: 2024-09-18T01:08:30.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mkdir test-primary10 2024-09-18T01:08:30.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo mount /dev/nbd9 test-primary10 2024-09-18T01:08:30.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:30.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:08:30.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:30.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:30.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:30.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:08:30.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3004 2024-09-18T01:08:30.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3005 2024-09-18T01:08:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:30.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:30.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ whoami 2024-09-18T01:08:30.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo chown ubuntu test-primary10 2024-09-18T01:08:30.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ SNAP_PIDS+=($!) 2024-09-18T01:08:30.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ take_mirror_snapshots cluster1 mirror image-primary10 2024-09-18T01:08:30.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ WORKLOAD_PIDS+=($!) 2024-09-18T01:08:30.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:08:30.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 130912 2024-09-18T01:08:30.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ slow_untar_workload test-primary10 2024-09-18T01:08:30.316 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mountpt=test-primary10 2024-09-18T01:08:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cp linux-5.4.tar.gz test-primary10 2024-09-18T01:08:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:08:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:30.317 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:08:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:08:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:30.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:08:30.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local ret=0 2024-09-18T01:08:30.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ timeout 5m bash -c 'zcat test-primary10/linux-5.4.tar.gz | pv -L 256K | tar xf - -C test-primary10' 2024-09-18T01:08:31.153 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:08:31.148+0000 7fd4c158b640 -1 reset not still connected to 0x564358707450 2024-09-18T01:08:31.153 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:08:31.148+0000 7fd4c158b640 -1 reset not still connected to 0x5643587075f0 2024-09-18T01:08:31.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:31.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:08:31.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:31.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:31.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:08:31.154 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:08:31.155 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:08:31.152+0000 7fd4c158b640 -1 reset not still connected to 0x564358707d40 2024-09-18T01:08:31.155 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:08:31.152+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a340 2024-09-18T01:08:31.155 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:08:31.152+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T01:08:31.155 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:08:31.152+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T01:08:31.155 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:08:31.152+0000 7fd4c158b640 -1 reset not still connected to 0x5643642b1ad0 2024-09-18T01:08:31.155 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:08:31.152+0000 7fd4c158b640 -1 reset not still connected to 0x56437c16a000 2024-09-18T01:08:31.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:31.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:08:31.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:31.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:31.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:31.178 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:08:31.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3008 2024-09-18T01:08:31.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:31.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3010 2024-09-18T01:08:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:32.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3009 2024-09-18T01:08:32.248 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3011 2024-09-18T01:08:32.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:32.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:32.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:32.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:08:32.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:32.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:32.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:32.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:08:32.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:32.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:08:32.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:32.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:32.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:32.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:08:32.866 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:32.863+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T01:08:32.866 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:32.863+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T01:08:32.867 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:32.863+0000 7f849c00d640 -1 reset not still connected to 0x5629bbd416c0 2024-09-18T01:08:33.158 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3012 2024-09-18T01:08:33.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:33.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3013 2024-09-18T01:08:33.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:33.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:08:33.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:33.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:33.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:33.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:08:33.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:33.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:33.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:08:33.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:33.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:33.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:33.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:08:34.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3014 2024-09-18T01:08:34.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:34.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3015 2024-09-18T01:08:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:08:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:34.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:08:34.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:08:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:34.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:08:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3016 2024-09-18T01:08:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:35.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:08:35.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:35.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:35.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:35.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:08:35.266 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3017 2024-09-18T01:08:35.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:35.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:08:35.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:35.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:35.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:35.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:08:35.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:35.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:36.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:36.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:08:36.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:36.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:36.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:36.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:08:36.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:36.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:08:36.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:36.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:36.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:08:36.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:08:36.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3019 2024-09-18T01:08:36.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3018 2024-09-18T01:08:36.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:36.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:37.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3021 2024-09-18T01:08:37.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:37.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:08:37.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:37.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:37.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:37.254 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:08:37.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3020 2024-09-18T01:08:37.267 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:37.286 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:37.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:37.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:08:37.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:37.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:37.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:37.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:08:38.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3022 2024-09-18T01:08:38.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3023 2024-09-18T01:08:38.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:38.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:38.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:08:38.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:38.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:38.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:38.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:08:38.309 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:38.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:08:38.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:38.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:38.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:38.310 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:08:39.262 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3025 2024-09-18T01:08:39.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3024 2024-09-18T01:08:39.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:39.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:08:39.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:39.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:39.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:08:39.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:39.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:39.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:39.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:08:39.311 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:39.307+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692a90 2024-09-18T01:08:39.311 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:08:39.307+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T01:08:40.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:40.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:08:40.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:40.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:40.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:40.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:08:40.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3026 2024-09-18T01:08:40.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:40.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:40.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:08:40.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:40.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:40.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:40.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:08:40.303 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3027 2024-09-18T01:08:40.336 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:41.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:41.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:08:41.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:41.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:41.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:41.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:08:41.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:41.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:08:41.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:41.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:41.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:08:41.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:08:41.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3029 2024-09-18T01:08:41.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:41.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3028 2024-09-18T01:08:41.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:42.059 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:08:42.056+0000 7f52d7c6c640 -1 reset not still connected to 0x5578572c04e0 2024-09-18T01:08:42.059 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:08:42.056+0000 7f52d7c6c640 -1 reset not still connected to 0x55787a4de750 2024-09-18T01:08:42.059 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:08:42.056+0000 7f52d7c6c640 -1 reset not still connected to 0x55788bba9ee0 2024-09-18T01:08:42.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3031 2024-09-18T01:08:42.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:42.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:08:42.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:42.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:42.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:42.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:08:42.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3030 2024-09-18T01:08:42.291 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:42.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:42.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:08:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:08:42.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:43.171 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3033 2024-09-18T01:08:43.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3032 2024-09-18T01:08:43.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:43.237 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:43.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:43.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:08:43.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:43.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:43.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:43.289 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:08:43.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:43.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:08:43.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:43.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:43.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:43.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:08:43.914 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:08:43.912+0000 7f52d7c6c640 -1 reset not still connected to 0x5578572c04e0 2024-09-18T01:08:43.919 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:08:43.916+0000 7f52d7c6c640 -1 reset not still connected to 0x55787a4de750 2024-09-18T01:08:43.919 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:08:43.916+0000 7f52d7c6c640 -1 reset not still connected to 0x55788bba9ee0 2024-09-18T01:08:44.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:44.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:08:44.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:44.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:44.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:44.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:08:44.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3034 2024-09-18T01:08:44.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:44.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:08:44.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:44.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:44.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:08:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3035 2024-09-18T01:08:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:44.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:45.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:45.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:08:45.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:45.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:45.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:45.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:08:45.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3036 2024-09-18T01:08:45.311 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3037 2024-09-18T01:08:45.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:45.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:08:45.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:45.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:45.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:45.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:08:45.329 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:45.330 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:46.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:46.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:08:46.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:46.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:46.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:08:46.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:08:46.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:46.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:08:46.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:46.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:46.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:46.239 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:08:46.315 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3039 2024-09-18T01:08:46.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:46.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3038 2024-09-18T01:08:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:47.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:47.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:08:47.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:47.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:47.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:47.325 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:08:47.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3040 2024-09-18T01:08:47.341 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3041 2024-09-18T01:08:47.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:47.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:47.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:08:47.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:47.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:47.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:47.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:08:47.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:48.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3042 2024-09-18T01:08:48.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:48.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3043 2024-09-18T01:08:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:08:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:48.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:08:48.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:48.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:08:48.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:48.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:48.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:48.333 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:08:48.936 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:08:48.931+0000 7f52d7c6c640 -1 reset not still connected to 0x5578427f8c30 2024-09-18T01:08:48.936 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:08:48.931+0000 7f52d7c6c640 -1 reset not still connected to 0x55784678b040 2024-09-18T01:08:48.938 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:08:48.931+0000 7f52d7c6c640 -1 reset not still connected to 0x5578572c04e0 2024-09-18T01:08:48.938 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:08:48.935+0000 7f52d7c6c640 -1 reset not still connected to 0x55787a4de750 2024-09-18T01:08:49.339 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3044 2024-09-18T01:08:49.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:49.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:08:49.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:49.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:49.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:49.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:08:49.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3045 2024-09-18T01:08:49.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:49.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:08:49.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:49.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:49.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:49.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:08:49.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:49.432 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:50.323 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3046 2024-09-18T01:08:50.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3047 2024-09-18T01:08:50.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:50.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:50.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:08:50.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:50.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:50.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:50.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:08:50.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:50.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:50.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:08:50.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:50.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:50.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:50.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:08:50.922 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:08:50.915+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T01:08:50.923 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:08:50.915+0000 7fd4c158b640 -1 reset not still connected to 0x5643642b1ad0 2024-09-18T01:08:50.924 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:08:50.915+0000 7fd4c158b640 -1 reset not still connected to 0x564364a0a5b0 2024-09-18T01:08:50.938 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:08:50.935+0000 7fd4c158b640 -1 reset not still connected to 0x56437c16a000 2024-09-18T01:08:51.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:51.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:08:51.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:51.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:51.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:08:51.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:08:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:08:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:51.224 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:08:51.324 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3048 2024-09-18T01:08:51.343 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:51.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3049 2024-09-18T01:08:51.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:52.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3051 2024-09-18T01:08:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:08:52.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:52.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:52.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:52.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:08:52.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3050 2024-09-18T01:08:52.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:52.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:08:53.156 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3052 2024-09-18T01:08:53.181 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3053 2024-09-18T01:08:53.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:53.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:53.354 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:08:53.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:53.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:53.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:53.355 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:08:53.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:53.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:08:53.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:53.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:53.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:53.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:08:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3055 2024-09-18T01:08:54.176 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3054 2024-09-18T01:08:54.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:54.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:08:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:54.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:54.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:54.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:08:54.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:54.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:08:55.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3057 2024-09-18T01:08:55.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3056 2024-09-18T01:08:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:55.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:55.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:08:55.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:55.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:55.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:08:55.379 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:08:55.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:55.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:55.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:08:55.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:55.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:55.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:08:55.421 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:08:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:08:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:08:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:08:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:08:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:08:56.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:08:56.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3058 2024-09-18T01:08:56.350 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3059 2024-09-18T01:08:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:56.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:57.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:57.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:08:57.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:57.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:57.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:57.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:08:57.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:08:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:08:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:08:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:08:57.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3060 2024-09-18T01:08:57.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3061 2024-09-18T01:08:57.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:57.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:58.163 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3064 2024-09-18T01:08:58.180 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3065 2024-09-18T01:08:58.184 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3062 2024-09-18T01:08:58.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3063 2024-09-18T01:08:58.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:58.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:58.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:58.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:08:58.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:58.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:08:58.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:58.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:58.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:08:58.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:08:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:08:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:08:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:08:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:08:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:08:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:08:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:08:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:08:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:08:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:08:59.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:08:59.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:00.371 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3066 2024-09-18T01:09:00.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3067 2024-09-18T01:09:00.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:00.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:09:00.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:00.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:00.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:09:00.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:00.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:09:00.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:09:00.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:00.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:00.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:09:00.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:09:00.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:00.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:09:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:09:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:09:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:01.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:09:01.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:01.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:01.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:09:01.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:09:01.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:01.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:09:01.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:01.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:01.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:09:01.208 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:09:01.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:01.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:09:01.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:01.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:01.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:09:01.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:09:01.378 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3068 2024-09-18T01:09:01.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:01.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3069 2024-09-18T01:09:01.420 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:02.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3071 2024-09-18T01:09:02.412 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3072 2024-09-18T01:09:02.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:02.430 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3070 2024-09-18T01:09:02.436 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:02.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:03.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3073 2024-09-18T01:09:03.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3074 2024-09-18T01:09:03.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:03.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3075 2024-09-18T01:09:03.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:03.228 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:03.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:03.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:09:03.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:03.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:03.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:09:03.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:09:03.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:03.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:09:03.427 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:03.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:03.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:09:03.428 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:09:04.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:04.406 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:09:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:09:04.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:04.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:04.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:09:04.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:04.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:04.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:09:04.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:09:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:09:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:09:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:09:05.435 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3076 2024-09-18T01:09:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:09:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:05.438 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:09:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:09:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3077 2024-09-18T01:09:05.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:05.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:09:05.446 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:05.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:05.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:09:05.447 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:09:05.454 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:05.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:06.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:06.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:09:06.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:06.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:06.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:09:06.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:09:06.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:09:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:09:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:09:06.431 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3079 2024-09-18T01:09:06.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3078 2024-09-18T01:09:06.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:06.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:07.261 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:09:07.259+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T01:09:07.261 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:09:07.259+0000 7fd4c158b640 -1 reset not still connected to 0x5643642b1ad0 2024-09-18T01:09:07.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3085 2024-09-18T01:09:07.422 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:07.434 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3081 2024-09-18T01:09:07.437 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3082 2024-09-18T01:09:07.439 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3080 2024-09-18T01:09:07.444 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3083 2024-09-18T01:09:07.456 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:07.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3084 2024-09-18T01:09:07.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:07.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:07.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:07.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:08.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:08.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:09:08.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:08.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:08.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:09:08.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:09:08.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:08.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:09:08.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:08.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:08.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:09:08.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:09:09.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:09.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:09:09.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:09.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:09.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:09:09.470 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:09:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:09:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:09:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:10.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:10.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:09:10.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:10.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:10.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:09:10.424 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:09:10.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:10.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:09:10.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:10.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:10.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:09:10.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:09:10.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3086 2024-09-18T01:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:09:10.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:09:10.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:10.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:09:10.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:10.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:10.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:09:10.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:09:10.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:10.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:09:10.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:10.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:10.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:09:10.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:09:10.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:10.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:09:10.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:10.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:10.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:09:10.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:09:10.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3087 2024-09-18T01:09:10.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:10.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:11.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3089 2024-09-18T01:09:11.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3088 2024-09-18T01:09:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:11.520 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3092 2024-09-18T01:09:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3095 2024-09-18T01:09:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3094 2024-09-18T01:09:12.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:12.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3091 2024-09-18T01:09:12.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3090 2024-09-18T01:09:12.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3093 2024-09-18T01:09:12.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:12.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:12.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:12.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:12.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:13.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:13.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:09:13.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:13.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:13.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:09:13.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:09:13.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:13.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:09:13.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:13.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:13.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:09:13.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:09:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:09:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:14.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:09:14.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:14.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:14.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:09:14.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:14.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:14.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:09:14.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:09:15.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:15.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:09:15.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:15.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:15.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:09:15.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:09:15.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3096 2024-09-18T01:09:15.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:15.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:09:15.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:15.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:15.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:09:15.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:09:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:09:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:09:15.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:09:15.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:15.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:09:15.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:15.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:15.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:09:15.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:09:15.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:15.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:09:15.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:15.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:15.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:09:15.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:09:15.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:15.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:15.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:09:15.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:15.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:15.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:09:15.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:09:15.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3097 2024-09-18T01:09:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:16.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3099 2024-09-18T01:09:16.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3098 2024-09-18T01:09:16.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:16.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:17.117 INFO:tasks.ceph.cluster2.osd.1.smithi049.stderr:2024-09-18T01:09:17.103+0000 7fbbe8b19640 -1 reset not still connected to 0x561a0cb8b930 2024-09-18T01:09:17.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3102 2024-09-18T01:09:17.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3105 2024-09-18T01:09:17.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3103 2024-09-18T01:09:17.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3104 2024-09-18T01:09:17.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3100 2024-09-18T01:09:17.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3101 2024-09-18T01:09:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:17.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:17.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:17.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:17.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:18.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:18.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:09:18.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:18.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:18.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:09:18.495 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:09:18.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:18.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:09:18.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:18.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:18.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:09:18.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:09:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:09:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:09:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:09:19.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:19.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:09:19.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:19.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:19.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:09:19.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:20.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3107 2024-09-18T01:09:20.522 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3106 2024-09-18T01:09:20.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:20.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:09:20.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:20.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:20.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:09:20.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:09:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:20.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:20.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:20.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:09:20.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:20.554 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:20.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:09:20.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:09:20.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:20.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:09:20.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:20.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:20.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:09:20.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:09:20.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:20.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:09:20.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:20.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:20.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:09:20.559 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:09:20.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:09:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:09:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:09:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:09:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:09:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:09:21.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3109 2024-09-18T01:09:21.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3108 2024-09-18T01:09:21.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:21.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:22.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3110 2024-09-18T01:09:22.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3114 2024-09-18T01:09:22.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3111 2024-09-18T01:09:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3115 2024-09-18T01:09:22.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3113 2024-09-18T01:09:22.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:22.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:22.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:22.549 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:22.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3112 2024-09-18T01:09:22.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:09:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:09:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:09:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:09:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:09:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:09:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:09:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:24.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:09:24.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:09:24.571 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:24.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:09:24.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:24.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:24.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:09:24.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:25.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3117 2024-09-18T01:09:25.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3116 2024-09-18T01:09:25.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:25.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:25.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:09:25.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:09:25.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:25.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:25.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:25.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:25.544 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:09:25.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:09:25.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:09:25.545 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:09:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:09:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:09:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:09:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:09:25.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:09:25.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:09:25.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:25.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:09:25.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:25.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:25.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:09:25.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:09:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:25.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:25.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:09:25.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:25.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:25.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:09:25.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:09:26.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3119 2024-09-18T01:09:26.550 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3118 2024-09-18T01:09:26.555 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:27.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3123 2024-09-18T01:09:27.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3125 2024-09-18T01:09:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3124 2024-09-18T01:09:27.553 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3121 2024-09-18T01:09:27.556 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3120 2024-09-18T01:09:27.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3122 2024-09-18T01:09:27.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:27.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:27.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:27.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:27.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:28.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:09:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:09:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:09:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:09:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:09:28.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:09:29.557 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:29.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:09:29.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:29.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:29.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:09:29.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:09:29.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:29.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:09:29.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:29.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:29.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:09:29.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:30.547 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3126 2024-09-18T01:09:30.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3127 2024-09-18T01:09:30.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:30.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:09:30.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:30.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:30.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:09:30.566 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:09:30.567 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:30.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:30.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:09:30.568 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:30.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:30.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:09:30.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:09:30.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:30.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:09:30.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:30.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:30.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:09:30.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:09:30.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:30.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:09:30.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:30.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:30.579 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:09:30.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:09:30.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:30.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:09:30.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:30.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:30.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:09:30.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:09:30.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:30.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:09:30.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:30.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:30.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:09:30.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:09:30.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:31.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3128 2024-09-18T01:09:31.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3129 2024-09-18T01:09:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:31.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:32.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3133 2024-09-18T01:09:32.551 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3130 2024-09-18T01:09:32.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3131 2024-09-18T01:09:32.562 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:32.572 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3132 2024-09-18T01:09:32.574 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3134 2024-09-18T01:09:32.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:32.581 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:32.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:32.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3135 2024-09-18T01:09:32.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:32.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:33.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:33.569 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:09:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:09:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:09:33.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:33.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:09:33.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:33.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:33.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:09:33.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:09:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:09:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:09:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:09:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:09:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:09:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:35.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:35.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:09:35.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:35.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:35.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:09:35.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:09:35.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:35.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:09:35.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:35.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:35.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:09:35.578 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:09:35.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:35.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:09:35.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:35.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:35.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:09:35.583 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:09:35.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:35.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:09:35.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:35.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:35.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:09:35.584 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:09:35.596 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3136 2024-09-18T01:09:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3137 2024-09-18T01:09:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:09:35.600 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:35.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:35.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:09:35.601 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:09:35.609 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:35.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:09:35.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:35.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:35.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:09:35.610 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:09:35.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:35.629 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:36.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3138 2024-09-18T01:09:36.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:36.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3139 2024-09-18T01:09:36.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:37.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3145 2024-09-18T01:09:37.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3140 2024-09-18T01:09:37.599 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3141 2024-09-18T01:09:37.612 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3144 2024-09-18T01:09:37.620 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3142 2024-09-18T01:09:37.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:37.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3143 2024-09-18T01:09:37.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:37.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:37.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:37.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:37.648 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:38.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:38.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:09:38.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:38.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:38.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:09:38.618 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:09:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:09:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:09:38.631 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:09:39.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:39.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:09:39.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:39.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:39.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:09:39.619 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:09:39.675 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:39.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:09:39.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:39.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:39.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:09:39.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:40.632 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3146 2024-09-18T01:09:40.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3147 2024-09-18T01:09:40.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:40.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:09:40.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:40.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:40.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:09:40.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:09:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:09:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:09:40.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:09:40.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:40.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:09:40.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:40.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:40.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:09:40.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:09:40.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:40.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:09:40.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:40.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:40.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:09:40.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:09:40.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:40.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:09:40.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:40.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:40.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:09:40.643 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:09:40.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:40.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:09:40.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:40.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:40.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:09:40.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:09:40.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:40.663 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3149 2024-09-18T01:09:41.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3148 2024-09-18T01:09:41.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:41.657 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:41.672 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:09:41.665+0000 7f849c00d640 -1 reset not still connected to 0x5629b9713040 2024-09-18T01:09:42.604 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3152 2024-09-18T01:09:42.616 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3150 2024-09-18T01:09:42.621 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3154 2024-09-18T01:09:42.634 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3155 2024-09-18T01:09:42.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:42.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:42.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3153 2024-09-18T01:09:42.638 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:42.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:42.654 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:42.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3151 2024-09-18T01:09:42.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:09:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:09:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:09:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:09:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:09:43.665 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:09:44.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:44.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:09:44.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:44.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:44.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:09:44.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:09:44.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:44.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:09:44.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:44.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:44.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:09:44.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:45.639 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:45.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:09:45.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:45.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:45.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:09:45.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:09:45.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:45.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:09:45.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:45.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:45.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:09:45.640 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:09:45.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:45.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:09:45.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:45.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:45.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:09:45.641 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:09:45.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:45.650 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:09:45.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:45.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:45.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:09:45.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:09:45.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3157 2024-09-18T01:09:45.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:45.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:09:45.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:45.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:45.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:09:45.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:09:45.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3156 2024-09-18T01:09:45.676 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:45.680 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:45.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:09:45.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:45.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:45.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:09:45.681 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:09:45.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:46.671 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3159 2024-09-18T01:09:46.677 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3158 2024-09-18T01:09:46.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:46.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:47.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3163 2024-09-18T01:09:47.683 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3161 2024-09-18T01:09:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3162 2024-09-18T01:09:47.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3160 2024-09-18T01:09:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3164 2024-09-18T01:09:47.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:47.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3165 2024-09-18T01:09:47.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:47.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:47.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:47.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:47.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:09:48.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:48.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:48.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:09:48.679 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:09:48.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:48.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:09:48.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:48.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:48.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:09:48.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:09:49.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:49.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:09:49.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:49.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:49.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:09:49.700 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:09:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:09:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:09:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:49.710 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:09:49.704+0000 7f849c00d640 -1 reset not still connected to 0x5629a8f8f930 2024-09-18T01:09:49.710 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:09:49.704+0000 7f849c00d640 -1 reset not still connected to 0x5629aa812f70 2024-09-18T01:09:49.710 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:09:49.704+0000 7f849c00d640 -1 reset not still connected to 0x5629af759ba0 2024-09-18T01:09:49.710 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:09:49.704+0000 7f849c00d640 -1 reset not still connected to 0x5629b9713040 2024-09-18T01:09:49.710 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:09:49.704+0000 7f849c00d640 -1 reset not still connected to 0x5629dc691380 2024-09-18T01:09:49.714 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:09:49.708+0000 7f849c00d640 -1 reset not still connected to 0x5629e34cc270 2024-09-18T01:09:50.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3167 2024-09-18T01:09:50.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3166 2024-09-18T01:09:50.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:50.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:09:50.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:50.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:50.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:09:50.696 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:09:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:09:50.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:50.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:50.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:09:50.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:09:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:09:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:09:50.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:09:50.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:50.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:09:50.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:50.707 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:50.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:09:50.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:09:50.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:09:50.717 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:50.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:50.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:50.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:09:50.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:50.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:50.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:09:50.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:09:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3168 2024-09-18T01:09:51.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:51.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3169 2024-09-18T01:09:51.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:52.678 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3172 2024-09-18T01:09:52.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3170 2024-09-18T01:09:52.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3175 2024-09-18T01:09:52.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3171 2024-09-18T01:09:52.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:52.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3174 2024-09-18T01:09:52.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:52.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3173 2024-09-18T01:09:52.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:52.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:52.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:53.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:53.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:09:53.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:53.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:53.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:09:53.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:09:53.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:53.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:09:53.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:53.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:53.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:09:53.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:09:54.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:54.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:09:54.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:54.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:54.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:09:54.713 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:09:54.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:54.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:09:54.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:54.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:54.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:09:54.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:55.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:55.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:09:55.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:55.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:55.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:09:55.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:09:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:09:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:09:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:09:55.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:55.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:09:55.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:55.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:55.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:09:55.721 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:09:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3176 2024-09-18T01:09:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:09:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:09:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:09:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:09:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:09:55.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:09:55.741 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3177 2024-09-18T01:09:55.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:55.742 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:09:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:09:55.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:09:55.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:55.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:56.723 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3179 2024-09-18T01:09:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3178 2024-09-18T01:09:56.740 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:56.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:57.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3185 2024-09-18T01:09:57.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3182 2024-09-18T01:09:57.754 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3181 2024-09-18T01:09:57.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3180 2024-09-18T01:09:57.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3184 2024-09-18T01:09:57.761 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3183 2024-09-18T01:09:57.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:57.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:57.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:57.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:57.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:09:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:09:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:09:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:09:58.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:58.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:09:58.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:58.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:58.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:09:58.766 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:09:59.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:59.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:09:59.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:59.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:59.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:09:59.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:09:59.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:09:59.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:09:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:09:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:09:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:09:59.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:10:00.735 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3187 2024-09-18T01:10:00.743 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3186 2024-09-18T01:10:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:00.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:00.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:00.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:10:00.774 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:00.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:00.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:10:00.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:10:00.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:00.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:10:00.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:00.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:00.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:10:00.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:10:00.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:00.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:10:00.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:00.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:00.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:10:00.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:10:00.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:00.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:10:00.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:00.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:00.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:00.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:10:00.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:10:00.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:10:00.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:00.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:00.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:10:00.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:10:00.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:00.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:10:00.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:00.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:00.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:10:00.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:10:01.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3188 2024-09-18T01:10:01.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3189 2024-09-18T01:10:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:01.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:02.762 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3191 2024-09-18T01:10:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3190 2024-09-18T01:10:02.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3194 2024-09-18T01:10:02.777 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3195 2024-09-18T01:10:02.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3193 2024-09-18T01:10:02.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3192 2024-09-18T01:10:02.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:02.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:02.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:02.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:02.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:02.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:03.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:03.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:10:03.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:03.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:03.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:10:03.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:10:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:10:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:10:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:10:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:10:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:10:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:10:04.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:04.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:10:04.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:04.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:04.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:10:04.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:10:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:10:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:10:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:10:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:10:05.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:05.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:05.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:10:05.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:10:05.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:05.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:10:05.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:05.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:05.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:10:05.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:10:05.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:05.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:10:05.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:05.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:05.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:10:05.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:10:05.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3196 2024-09-18T01:10:05.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:10:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:10:05.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:10:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3197 2024-09-18T01:10:05.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:05.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:10:05.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:05.824 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:05.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:10:05.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:10:05.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:05.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3198 2024-09-18T01:10:06.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3199 2024-09-18T01:10:06.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:06.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:07.763 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3200 2024-09-18T01:10:07.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3204 2024-09-18T01:10:07.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3205 2024-09-18T01:10:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:07.811 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3202 2024-09-18T01:10:07.818 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3201 2024-09-18T01:10:07.819 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:07.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:07.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3203 2024-09-18T01:10:07.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:07.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:07.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:08.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:08.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:10:08.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:08.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:08.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:10:08.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:10:08.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:08.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:10:08.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:08.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:08.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:10:08.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:10:09.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:10:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:10:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:10:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:10:10.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:10.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:10:10.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:10.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:10.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:10:10.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:10:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:10.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:10:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:10:10.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:10:10.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:10.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:10:10.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:10.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:10.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:10:10.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:10:10.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3207 2024-09-18T01:10:10.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3206 2024-09-18T01:10:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:10:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:10:10.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:10.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:10.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:10.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:10.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:10:10.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:10:10.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:10:10.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:10:10.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:10.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:10.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:10.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:10:10.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:10.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:10.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:10:10.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:10:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3208 2024-09-18T01:10:11.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3209 2024-09-18T01:10:11.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:11.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:12.807 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3210 2024-09-18T01:10:12.815 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3213 2024-09-18T01:10:12.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3212 2024-09-18T01:10:12.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:12.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:12.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3211 2024-09-18T01:10:12.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3214 2024-09-18T01:10:12.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:12.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:12.863 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3215 2024-09-18T01:10:12.873 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:12.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:13.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:13.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:10:13.849 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:13.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:13.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:10:13.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:10:13.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:13.850 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:10:13.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:13.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:13.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:10:13.851 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:10:14.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:14.877 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:10:14.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:14.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:14.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:10:14.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:10:14.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:14.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:10:14.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:14.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:14.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:10:14.916 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:10:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:15.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:10:15.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:15.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:15.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:10:15.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:10:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:10:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:15.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:15.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:10:15.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:10:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3216 2024-09-18T01:10:15.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:15.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:10:15.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:15.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:15.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:10:15.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:10:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:10:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:15.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:10:15.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:10:15.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3217 2024-09-18T01:10:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:10:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:10:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:10:15.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:15.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:10:15.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:15.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:15.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:10:15.879 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:10:15.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:15.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:16.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3219 2024-09-18T01:10:16.880 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3218 2024-09-18T01:10:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:17.865 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3220 2024-09-18T01:10:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3225 2024-09-18T01:10:17.867 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3223 2024-09-18T01:10:17.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3222 2024-09-18T01:10:17.871 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3224 2024-09-18T01:10:17.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3221 2024-09-18T01:10:17.882 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:17.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:17.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:17.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:17.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:17.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:18.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:10:18.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:18.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:10:18.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:18.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:18.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:10:18.898 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:10:19.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:19.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:10:19.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:19.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:19.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:10:19.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:10:19.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:19.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:10:19.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:19.905 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:10:19.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:10:20.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3226 2024-09-18T01:10:20.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:20.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:10:20.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:20.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:20.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:10:20.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:10:20.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:20.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:10:20.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:20.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:20.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:10:20.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:10:20.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:20.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:10:20.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:20.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:20.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:10:20.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:10:20.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:10:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:20.899 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:10:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:10:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:10:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:10:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:10:20.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:20.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:10:20.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:20.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:20.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:10:20.903 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:10:20.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3227 2024-09-18T01:10:20.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:21.908 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3229 2024-09-18T01:10:21.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3228 2024-09-18T01:10:21.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:21.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:22.881 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3232 2024-09-18T01:10:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3231 2024-09-18T01:10:22.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3230 2024-09-18T01:10:22.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3234 2024-09-18T01:10:22.901 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3235 2024-09-18T01:10:22.909 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3233 2024-09-18T01:10:22.910 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:22.917 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:22.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:22.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:22.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:22.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:10:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:10:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:10:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:10:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:10:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:10:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:10:24.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:24.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:24.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:10:24.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:10:24.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:24.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:10:24.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:24.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:24.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:10:24.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:10:25.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3237 2024-09-18T01:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3236 2024-09-18T01:10:25.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:25.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:10:25.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:10:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:10:25.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:10:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:25.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:25.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:10:25.920 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:10:25.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:10:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:10:25.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:10:25.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:25.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:25.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:10:25.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:25.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:25.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:10:25.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:10:25.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:25.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:10:25.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:25.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:25.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:10:25.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:10:25.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:25.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:10:25.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:25.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:25.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:10:25.936 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:10:26.894 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3239 2024-09-18T01:10:26.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:26.930 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3238 2024-09-18T01:10:26.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:27.834 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:10:27.827+0000 7f849c00d640 -1 reset not still connected to 0x5629d9a541a0 2024-09-18T01:10:27.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3245 2024-09-18T01:10:27.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3240 2024-09-18T01:10:27.900 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3241 2024-09-18T01:10:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3242 2024-09-18T01:10:27.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3244 2024-09-18T01:10:27.922 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:27.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:27.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:27.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3243 2024-09-18T01:10:27.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:27.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:27.943 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:28.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:28.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:10:28.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:28.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:28.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:10:28.915 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:10:28.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:28.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:10:28.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:28.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:28.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:10:28.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:10:29.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:29.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:10:29.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:29.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:29.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:10:29.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:10:29.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:29.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:10:29.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:29.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:29.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:10:29.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:10:30.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:30.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:10:30.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:30.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:30.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:10:30.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:10:30.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:30.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:10:30.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:30.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:30.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:10:30.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:10:30.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:30.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:10:30.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:30.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:30.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:10:30.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:10:30.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:30.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:10:30.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:30.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:30.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:10:30.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:10:30.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:30.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:10:30.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:30.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:30.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:10:30.935 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:10:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:10:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:10:30.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:10:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3247 2024-09-18T01:10:30.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3246 2024-09-18T01:10:30.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3248 2024-09-18T01:10:31.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3249 2024-09-18T01:10:31.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:31.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:32.941 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3252 2024-09-18T01:10:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3251 2024-09-18T01:10:32.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3254 2024-09-18T01:10:32.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3255 2024-09-18T01:10:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3253 2024-09-18T01:10:32.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3250 2024-09-18T01:10:32.975 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:32.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:33.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:33.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:33.005 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:33.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:33.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:33.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:10:33.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:33.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:33.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:10:33.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:10:33.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:33.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:10:33.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:33.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:33.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:10:33.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:10:34.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:34.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:10:34.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:34.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:34.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:10:34.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:10:34.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:34.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:10:34.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:34.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:34.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:10:34.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:10:35.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:35.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary1 2024-09-18T01:10:35.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:35.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:35.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:10:35.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary1 2024-09-18T01:10:36.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:36.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:10:36.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:36.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:36.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:10:36.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:10:36.001 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:10:35.992+0000 7f52d7c6c640 -1 reset not still connected to 0x55787a4de750 2024-09-18T01:10:36.001 INFO:tasks.ceph.cluster2.osd.0.smithi049.stderr:2024-09-18T01:10:35.992+0000 7f52d7c6c640 -1 reset not still connected to 0x55788bba9ee0 2024-09-18T01:10:36.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:36.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:10:36.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:36.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:36.007 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:10:36.008 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:10:36.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:36.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:10:36.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:36.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:36.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:10:36.009 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:10:36.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:36.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:10:36.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:36.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:36.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:10:36.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:10:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:10:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:10:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:10:36.169 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3258 2024-09-18T01:10:36.174 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3257 2024-09-18T01:10:36.188 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3256 2024-09-18T01:10:36.190 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3259 2024-09-18T01:10:36.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:36.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:36.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:36.217 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:37.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3262 2024-09-18T01:10:37.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:37.240 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3261 2024-09-18T01:10:37.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3260 2024-09-18T01:10:37.251 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:37.261 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:38.187 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3265 2024-09-18T01:10:38.189 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3263 2024-09-18T01:10:38.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3264 2024-09-18T01:10:38.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:38.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:38.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:39.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:39.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:10:39.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:39.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:39.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:10:39.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:10:39.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:39.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary2 2024-09-18T01:10:39.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:39.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:39.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:10:39.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary2 2024-09-18T01:10:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:10:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:10:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:10:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:10:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:10:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:10:40.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:40.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:10:40.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:40.222 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:40.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:10:40.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:10:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:10:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:10:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:10:40.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:10:40.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 131045 2024-09-18T01:10:40.440 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3266 2024-09-18T01:10:40.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3269 2024-09-18T01:10:40.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3268 2024-09-18T01:10:40.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3267 2024-09-18T01:10:40.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:40.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary3 2024-09-18T01:10:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:10:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary3 2024-09-18T01:10:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary4 2024-09-18T01:10:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:10:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary4 2024-09-18T01:10:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:10:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:10:41.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:10:41.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3271 2024-09-18T01:10:41.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3270 2024-09-18T01:10:41.458 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:41.467 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:42.455 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3274 2024-09-18T01:10:42.457 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3273 2024-09-18T01:10:42.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3272 2024-09-18T01:10:42.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:42.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:42.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:43.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:10:43.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 131188 2024-09-18T01:10:43.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:43.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary5 2024-09-18T01:10:43.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:43.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:43.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:10:43.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary5 2024-09-18T01:10:43.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:43.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:10:43.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:43.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:43.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:43.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:10:43.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:10:43.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:10:43.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:43.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:43.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:10:43.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:10:44.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:44.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:10:44.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:44.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:44.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:10:44.462 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:10:44.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:44.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:10:44.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:44.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:44.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:10:44.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:10:45.448 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3276 2024-09-18T01:10:45.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3277 2024-09-18T01:10:45.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:45.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:10:45.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 131381 2024-09-18T01:10:45.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3275 2024-09-18T01:10:45.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:45.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary6 2024-09-18T01:10:45.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:45.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:45.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:10:45.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary6 2024-09-18T01:10:45.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:10:45.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 131554 2024-09-18T01:10:45.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:45.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:46.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3278 2024-09-18T01:10:46.468 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3279 2024-09-18T01:10:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:46.483 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:47.459 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3280 2024-09-18T01:10:47.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:48.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:48.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:10:48.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:48.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:48.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:10:48.466 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:10:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary7 2024-09-18T01:10:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:10:48.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary7 2024-09-18T01:10:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:10:48.514 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 131713 2024-09-18T01:10:49.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:49.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary9 2024-09-18T01:10:49.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:49.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:49.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:10:49.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary9 2024-09-18T01:10:49.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:49.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary8 2024-09-18T01:10:49.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:49.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:49.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:10:49.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary8 2024-09-18T01:10:50.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:10:50.480 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 131909 2024-09-18T01:10:50.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3282 2024-09-18T01:10:50.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3281 2024-09-18T01:10:50.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:50.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:51.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3284 2024-09-18T01:10:51.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3283 2024-09-18T01:10:51.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:51.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:53.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for i in {1..30} 2024-09-18T01:10:53.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ mirror_image_snapshot cluster1 mirror image-primary10 2024-09-18T01:10:53.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:10:53.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:10:53.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:10:53.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image-primary10 2024-09-18T01:10:53.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:10:53.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 132104 2024-09-18T01:10:54.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:10:54.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 132261 2024-09-18T01:10:54.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${SNAP_PIDS[@]} 2024-09-18T01:10:54.523 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 132496 2024-09-18T01:10:55.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Snapshot ID: 3285 2024-09-18T01:10:55.247 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 3 2024-09-18T01:10:58.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T01:10:58.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:10:58.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 130913 2024-09-18T01:13:13.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:13:13.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:13:13.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:13:13.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 131046 2024-09-18T01:13:14.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:13:14.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:13:14.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:13:14.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 131189 2024-09-18T01:13:16.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:13:16.182 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:13:16.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:13:16.183 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 131382 2024-09-18T01:13:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:13:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:13:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:13:18.146 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 131555 2024-09-18T01:13:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:13:20.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:13:20.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:13:20.383 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 131714 2024-09-18T01:13:22.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:13:22.241 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:13:22.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:13:22.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 131910 2024-09-18T01:13:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:13:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:13:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:13:24.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 132105 2024-09-18T01:13:26.607 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:13:26.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:13:26.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:13:26.608 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 132262 2024-09-18T01:13:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:13:28.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:13:28.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${WORKLOAD_PIDS[@]} 2024-09-18T01:13:28.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 132497 2024-09-18T01:13:30.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ret=124 2024-09-18T01:13:30.525 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( ret != 124 )) 2024-09-18T01:13:30.526 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T01:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 1 2024-09-18T01:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd0 2024-09-18T01:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary1 2024-09-18T01:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary1 2024-09-18T01:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary1 2024-09-18T01:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 2 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd1 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary2 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary2 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary2 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 3 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd2 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary3 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary3 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:13:30.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary3 2024-09-18T01:13:30.529 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 4 2024-09-18T01:13:30.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd3 2024-09-18T01:13:30.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary4 2024-09-18T01:13:30.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary4 2024-09-18T01:13:30.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:13:30.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:13:30.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary4 2024-09-18T01:13:30.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:13:30.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 5 2024-09-18T01:13:30.530 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd4 2024-09-18T01:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary5 2024-09-18T01:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 6 2024-09-18T01:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary5 2024-09-18T01:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary5 2024-09-18T01:13:30.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd5 2024-09-18T01:13:30.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary6 2024-09-18T01:13:30.533 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 7 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary6 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary6 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd6 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary7 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary7 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary7 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:13:30.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:13:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:13:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ COMPARE_PIDS+=($!) 2024-09-18T01:13:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:13:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 138422 2024-09-18T01:13:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 10 2024-09-18T01:13:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd9 2024-09-18T01:13:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary10 2024-09-18T01:13:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary10 2024-09-18T01:13:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:13:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary10 2024-09-18T01:13:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 9 2024-09-18T01:13:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd8 2024-09-18T01:13:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary9 2024-09-18T01:13:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary9 2024-09-18T01:13:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:13:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary9 2024-09-18T01:13:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ compare_demoted_promoted_image 8 2024-09-18T01:13:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local dev=/dev/nbd7 2024-09-18T01:13:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local img=image-primary8 2024-09-18T01:13:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local mntpt=test-primary8 2024-09-18T01:13:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local demote_md5 promote_md5 2024-09-18T01:13:30.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo umount test-primary8 2024-09-18T01:13:31.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T01:13:31.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:13:31.028 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T01:13:31.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:13:31.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T01:13:31.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:13:31.112 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T01:13:31.113 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:13:31.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T01:13:31.114 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:13:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T01:13:31.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:13:31.127 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T01:13:31.131 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:13:31.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:13:31.134 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T01:13:31.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:13:31.139 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T01:13:31.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T01:13:31.157 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:14:30.356 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=88340d70a35dd3634e58c9ad1214effc 2024-09-18T01:14:30.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary1 2024-09-18T01:14:30.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=77c91e0d9e3cc0edd3e96d2cc9698045 2024-09-18T01:14:30.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary3 2024-09-18T01:14:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=1d7b78367e6688efc34bce166fc6879a 2024-09-18T01:14:31.441 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary8 2024-09-18T01:14:31.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=06c7e007505e08ac519a8e9e1072cfb9 2024-09-18T01:14:31.672 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary9 2024-09-18T01:14:31.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary1 2024-09-18T01:14:31.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:31.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:31.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:14:31.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary1 2024-09-18T01:14:31.997 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary3 2024-09-18T01:14:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:14:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary3 2024-09-18T01:14:32.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=b0f32cd4bff1e0b1e816bb401888bd37 2024-09-18T01:14:32.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary4 2024-09-18T01:14:32.918 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary8 2024-09-18T01:14:32.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:32.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:32.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:14:32.919 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary8 2024-09-18T01:14:32.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=60428a142d540eee7ad53d4e4cb767e7 2024-09-18T01:14:32.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary10 2024-09-18T01:14:32.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary9 2024-09-18T01:14:32.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:32.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:32.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:14:32.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary9 2024-09-18T01:14:33.326 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary4 2024-09-18T01:14:33.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:33.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:33.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:14:33.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary4 2024-09-18T01:14:33.484 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=0d1978ecf45d9f238aaf370a281abcfb 2024-09-18T01:14:33.485 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary6 2024-09-18T01:14:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=7d0fe805a523bb4cddbac8609b410d87 2024-09-18T01:14:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary2 2024-09-18T01:14:33.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=5c17a00ccaded73adbf2ff3f23d90d13 2024-09-18T01:14:33.765 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary5 2024-09-18T01:14:33.946 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:14:33.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:14:33.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+unknown 2024-09-18T01:14:33.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:33.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:33.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:14:33.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:33.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:33.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:33.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:33.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:33.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+unknown 2024-09-18T01:14:33.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:33.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:33.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:14:33.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:33.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:33.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:33.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:33.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:34.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary10 2024-09-18T01:14:34.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:34.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:34.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:14:34.000 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary10 2024-09-18T01:14:34.263 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary6 2024-09-18T01:14:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:14:34.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary6 2024-09-18T01:14:34.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary2 2024-09-18T01:14:34.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:34.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:34.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:14:34.338 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary2 2024-09-18T01:14:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary5 2024-09-18T01:14:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:14:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary5 2024-09-18T01:14:34.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:14:34.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:14:34.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:14:34.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+unknown '' '' 2024-09-18T01:14:34.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:34.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:34.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:14:34.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:34.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:34.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:34.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:34.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:34.968 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T01:14:34.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+unknown 2024-09-18T01:14:34.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:34.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:34.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:14:34.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:34.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:34.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:34.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:34.977 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:34.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+unknown 2024-09-18T01:14:34.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:34.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:34.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:14:34.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:34.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:34.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:34.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:34.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+unknown 2024-09-18T01:14:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:14:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+unknown '' '' 2024-09-18T01:14:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:14:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:34.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T01:14:35.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T01:14:35.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: d0f03b0b-70ae-444b-9369-278125560577 2024-09-18T01:14:35.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:35.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:35.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:35.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:35.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:35.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:35.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:35.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:35.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34' 2024-09-18T01:14:35.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:35.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T01:14:35.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 821edf46-8d24-4ea2-bb84-66b4b58ea23a 2024-09-18T01:14:35.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:35.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:35.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:35.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:35.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:35.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:35.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:35.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:35.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34' 2024-09-18T01:14:35.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:35.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:35.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:35.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:35.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:35.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:35.083 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:35.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:35.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:35.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:35.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:35.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:35.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+unknown '2024-09-18 01:14:34' 2024-09-18T01:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+unknown '2024-09-18 01:14:34' 2024-09-18T01:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:34' 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:34' 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:35.105 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:35.306 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:34image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1710080.0,"last_snapshot_bytes":1732608,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621854,"remote_snapshot_timestamp":1726621854,"replay_state":"idle"}2024-09-18 01:14:09image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2613248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621839,"remote_snapshot_timestamp":1726621839,"replay_state":"idle"}2024-09-18 01:14:09image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:34image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 01:14:35image-primary590242e82-997f-4088-b460-92f5b78a8509up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":4331520.0,"last_snapshot_bytes":4333568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621844,"remote_snapshot_timestamp":1726621844,"replay_state":"idle"}2024-09-18 01:14:09image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":5142528.0,"last_snapshot_bytes":2281472,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621846,"remote_snapshot_timestamp":1726621846,"replay_state":"idle"}2024-09-18 01:14:09image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2598912.0,"last_snapshot_bytes":237568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621849,"remote_snapshot_timestamp":1726621849,"replay_state":"idle"}2024-09-18 01:14:09image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 01:14:35image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35' 2024-09-18T01:14:35.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:35.307 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T01:14:35.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_md5=7b3d95e809aa30394eb8afe2b21aa632 2024-09-18T01:14:35.320 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster1 device unmap -t nbd mirror/image-primary7 2024-09-18T01:14:35.377 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:35.380 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T01:14:35.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:35.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:35.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:35.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:35.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:35.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:35.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary1 up+unknown 2024-09-18T01:14:35.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:35.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:35.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:14:35.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:35.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:35.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:35.399 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:35.400 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:35.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:34image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1710080.0,"last_snapshot_bytes":1732608,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621854,"remote_snapshot_timestamp":1726621854,"replay_state":"idle"}2024-09-18 01:14:09image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2613248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621839,"remote_snapshot_timestamp":1726621839,"replay_state":"idle"}2024-09-18 01:14:09image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:34image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35image-primary590242e82-997f-4088-b460-92f5b78a8509up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":4331520.0,"last_snapshot_bytes":4333568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621844,"remote_snapshot_timestamp":1726621844,"replay_state":"idle"}2024-09-18 01:14:09image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":5142528.0,"last_snapshot_bytes":2281472,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621846,"remote_snapshot_timestamp":1726621846,"replay_state":"idle"}2024-09-18 01:14:09image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2598912.0,"last_snapshot_bytes":237568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621849,"remote_snapshot_timestamp":1726621849,"replay_state":"idle"}2024-09-18 01:14:09image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35' 2024-09-18T01:14:35.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:35.521 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T01:14:35.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:35.524 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T01:14:35.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:35.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:35.532 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:35.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:35.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:35.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:35.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary3 up+unknown 2024-09-18T01:14:35.534 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:35.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:35.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:14:35.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:35.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:35.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:35.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:35.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:35.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:14:35.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:14:35.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:14:35.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+unknown 2024-09-18T01:14:35.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:35.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:35.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:14:35.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+unknown '' '' 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+unknown 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:35.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:35.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:14:35.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:35.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:35.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:35.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:35.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:35.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:35.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:35.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:35.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T01:14:35.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+unknown '' '' 2024-09-18T01:14:35.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:35.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:35.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:14:35.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:35.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:35.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:35.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:35.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:35.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T01:14:35.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+unknown 2024-09-18T01:14:35.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:14:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:35.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:35.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+unknown '' '' 2024-09-18T01:14:35.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:35.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:35.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:14:35.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:35.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:35.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:35.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:35.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:35.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T01:14:36.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T01:14:36.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: b5da1212-bf20-4dc9-bf59-f911a78837d4 2024-09-18T01:14:36.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:36.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:36.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:36.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:36.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:36.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:36.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:36.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:36.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:35' 2024-09-18T01:14:36.052 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:36.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:36.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:36.053 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:36.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:36.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:36.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:36.054 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:36.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:36.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:36.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:36.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:36.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T01:14:36.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: ee4c71cc-a845-47e1-b515-dffaef47a382 2024-09-18T01:14:36.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:36.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:36.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:36.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:36.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:36.056 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:36.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:36.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:36.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:35' 2024-09-18T01:14:36.057 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+unknown '2024-09-18 01:14:34' 2024-09-18T01:14:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:14:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:34' 2024-09-18T01:14:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:36.058 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:36.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:36.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:36.059 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:36.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:36.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:36.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:36.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:36.060 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:36.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:36.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:36.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T01:14:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8c413c2e-0975-4386-8c15-1b1c7c115008 2024-09-18T01:14:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:36.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:35' 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+unknown '2024-09-18 01:14:34' 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.063 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:34' 2024-09-18T01:14:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:36.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:36.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:36.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:36.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:36.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:36.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+unknown '2024-09-18 01:14:34' 2024-09-18T01:14:36.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:36.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:14:36.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:34' 2024-09-18T01:14:36.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:36.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:36.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ demote_image cluster1 mirror image-primary7 2024-09-18T01:14:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:14:36.133 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/image-primary7 2024-09-18T01:14:36.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:34image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1710080.0,"last_snapshot_bytes":1732608,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621854,"remote_snapshot_timestamp":1726621854,"replay_state":"idle"}2024-09-18 01:14:09image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2613248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621839,"remote_snapshot_timestamp":1726621839,"replay_state":"idle"}2024-09-18 01:14:09image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:34image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35image-primary590242e82-997f-4088-b460-92f5b78a8509up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":4331520.0,"last_snapshot_bytes":4333568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621844,"remote_snapshot_timestamp":1726621844,"replay_state":"idle"}2024-09-18 01:14:09image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":5142528.0,"last_snapshot_bytes":2281472,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621846,"remote_snapshot_timestamp":1726621846,"replay_state":"idle"}2024-09-18 01:14:09image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2598912.0,"last_snapshot_bytes":237568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621849,"remote_snapshot_timestamp":1726621849,"replay_state":"idle"}2024-09-18 01:14:09image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35' 2024-09-18T01:14:36.265 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:36.279 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:34image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1710080.0,"last_snapshot_bytes":1732608,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621854,"remote_snapshot_timestamp":1726621854,"replay_state":"idle"}2024-09-18 01:14:09image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2613248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621839,"remote_snapshot_timestamp":1726621839,"replay_state":"idle"}2024-09-18 01:14:09image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:34image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35image-primary590242e82-997f-4088-b460-92f5b78a8509up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":4331520.0,"last_snapshot_bytes":4333568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621844,"remote_snapshot_timestamp":1726621844,"replay_state":"idle"}2024-09-18 01:14:09image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 01:14:36image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2598912.0,"last_snapshot_bytes":237568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621849,"remote_snapshot_timestamp":1726621849,"replay_state":"idle"}2024-09-18 01:14:09image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35' 2024-09-18T01:14:36.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:36.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T01:14:36.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T01:14:36.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:34image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":1710080.0,"last_snapshot_bytes":1732608,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621854,"remote_snapshot_timestamp":1726621854,"replay_state":"idle"}2024-09-18 01:14:09image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2152448.0,"last_snapshot_bytes":2613248,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621839,"remote_snapshot_timestamp":1726621839,"replay_state":"idle"}2024-09-18 01:14:09image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:34image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35image-primary590242e82-997f-4088-b460-92f5b78a8509up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":4331520.0,"last_snapshot_bytes":4333568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621844,"remote_snapshot_timestamp":1726621844,"replay_state":"idle"}2024-09-18 01:14:09image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stopping_replayremote image demoted2024-09-18 01:14:36image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+stoppedlocal image is primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2598912.0,"last_snapshot_bytes":237568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621849,"remote_snapshot_timestamp":1726621849,"replay_state":"idle"}2024-09-18 01:14:09image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:34cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image demoted2024-09-18 01:14:35' 2024-09-18T01:14:36.280 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:36.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:36.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T01:14:36.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T01:14:36.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:36.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:36.287 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T01:14:36.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T01:14:36.290 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:36.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:36.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:36.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:36.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:36.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:36.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:36.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:36.294 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:36.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:36.295 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary4 up+unknown 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary8 up+unknown 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary9 up+unknown 2024-09-18T01:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:36.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary1 up+unknown '' '' 2024-09-18T01:14:36.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:36.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:14:36.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:36.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:36.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:36.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:36.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary1 2024-09-18T01:14:36.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T01:14:36.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: d0f03b0b-70ae-444b-9369-278125560577 2024-09-18T01:14:36.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:36.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:36.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:36.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:36.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:36.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:14:36.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34' 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary3 up+unknown '' '' 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:36.537 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary3 2024-09-18T01:14:36.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:36.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:36.538 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T01:14:36.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:36.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:36.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:36.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:36.539 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:36.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:36.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:36.540 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:36.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:36.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary1 up+unknown '2024-09-18 01:14:34' 2024-09-18T01:14:36.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:36.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:14:36.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:34' 2024-09-18T01:14:36.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:36.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:36.543 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:36.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T01:14:36.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 821edf46-8d24-4ea2-bb84-66b4b58ea23a 2024-09-18T01:14:36.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:36.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:36.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:36.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:36.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:36.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:14:36.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:36.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:36.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34' 2024-09-18T01:14:36.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:36.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T01:14:36.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:36.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:36.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:36.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:36.593 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:36.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:36.594 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:36.595 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:36.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:36.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary3 up+unknown '2024-09-18 01:14:34' 2024-09-18T01:14:36.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:36.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:14:36.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:34' 2024-09-18T01:14:36.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:36.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:36.597 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:36.646 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary590242e82-997f-4088-b460-92f5b78a8509up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":4331520.0,"last_snapshot_bytes":4333568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621844,"remote_snapshot_timestamp":1726621844,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 01:14:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2598912.0,"last_snapshot_bytes":237568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621849,"remote_snapshot_timestamp":1726621849,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 01:14:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34' 2024-09-18T01:14:36.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:36.647 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T01:14:36.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:36.649 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T01:14:36.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:36.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:36.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:36.653 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:36.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:36.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:36.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary1 2024-09-18T01:14:36.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:36.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:14:36.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:14:36.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary1 2024-09-18T01:14:36.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING8242614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary590242e82-997f-4088-b460-92f5b78a8509up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":4331520.0,"last_snapshot_bytes":4333568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621844,"remote_snapshot_timestamp":1726621844,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 01:14:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2598912.0,"last_snapshot_bytes":237568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621849,"remote_snapshot_timestamp":1726621849,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 01:14:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34' 2024-09-18T01:14:36.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:36.708 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T01:14:36.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:34' 2024-09-18T01:14:36.711 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T01:14:36.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:36.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:36.714 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:36.715 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:36.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:36.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:36.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary3 2024-09-18T01:14:36.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:36.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:14:36.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:14:36.716 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary3 2024-09-18T01:14:36.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:14:36.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+unknown 2024-09-18T01:14:36.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:36.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:14:36.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:36.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:36.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:36.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:36.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T01:14:36.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:36.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:14:36.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:36.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:36.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:36.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T01:14:36.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:36.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:14:36.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:36.983 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:36.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T01:14:36.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:36.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:36.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:36.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T01:14:36.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T01:14:36.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:36.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:36.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:14:36.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:36.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:36.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:36.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:36.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:36.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T01:14:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T01:14:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 68158950-cc3b-4cfd-bf00-d7829fce7865 2024-09-18T01:14:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:14:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:14:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:37.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:36' 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 01a5a6a4-a190-4d6c-9392-bbfa30f6ec37 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:37.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:36' 2024-09-18T01:14:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:14:37.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T01:14:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:14:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T01:14:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T01:14:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 977eb8ba-b7cf-46d7-bbaf-d2f345b9d999 2024-09-18T01:14:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:14:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:14:37.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:37.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:37.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:37.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:37.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:37.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:37.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:36' 2024-09-18T01:14:37.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:14:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:37.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T01:14:37.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:14:37.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:14:37.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:14:37.282 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary1 2024-09-18T01:14:37.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary8 up+unknown '' '' 2024-09-18T01:14:37.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:37.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:37.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:14:37.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:37.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:37.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:37.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:37.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:37.297 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary8 2024-09-18T01:14:37.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary9 up+unknown '' '' 2024-09-18T01:14:37.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:37.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:37.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:14:37.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:37.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:37.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:37.300 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:37.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary4 up+unknown '' '' 2024-09-18T01:14:37.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:37.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:37.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:14:37.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:37.304 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:37.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:37.305 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:37.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:37.321 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary4 2024-09-18T01:14:37.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:37.327 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary9 2024-09-18T01:14:37.331 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:14:37.332 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary3 2024-09-18T01:14:37.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T01:14:37.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: ee4c71cc-a845-47e1-b515-dffaef47a382 2024-09-18T01:14:37.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:37.471 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:37.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:37.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:35 2024-09-18T01:14:37.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:37.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:14:37.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:37.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:37.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34' 2024-09-18T01:14:37.472 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:37.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:37.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:37.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T01:14:37.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:37.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:37.476 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:37.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:37.478 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:37.479 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:35' 2024-09-18T01:14:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary4 up+unknown '2024-09-18 01:14:35' 2024-09-18T01:14:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:14:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:35' 2024-09-18T01:14:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T01:14:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8c413c2e-0975-4386-8c15-1b1c7c115008 2024-09-18T01:14:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:35 2024-09-18T01:14:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:14:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34' 2024-09-18T01:14:37.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:37.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:37.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:37.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T01:14:37.489 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:37.490 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:37.491 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:37.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:37.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:37.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:37.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:37.494 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:37.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:35' 2024-09-18T01:14:37.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary8 up+unknown '2024-09-18 01:14:35' 2024-09-18T01:14:37.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:37.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:37.496 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:14:37.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:37.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:35' 2024-09-18T01:14:37.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:37.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:37.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T01:14:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: b5da1212-bf20-4dc9-bf59-f911a78837d4 2024-09-18T01:14:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:35 2024-09-18T01:14:37.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:14:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34' 2024-09-18T01:14:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:37.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:37.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image demoted 2024-09-18T01:14:37.507 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:37.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:37.508 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:37.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:37.509 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:37.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:37.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:37.510 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:37.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:35' 2024-09-18T01:14:37.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary9 up+unknown '2024-09-18 01:14:35' 2024-09-18T01:14:37.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:37.512 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:37.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:14:37.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:37.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:35' 2024-09-18T01:14:37.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:37.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:37.513 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:37.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd0 2024-09-18T01:14:37.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd0 2024-09-18T01:14:37.542 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:14:37.580 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd1 2024-09-18T01:14:37.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd1 2024-09-18T01:14:37.582 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:14:37.728 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary590242e82-997f-4088-b460-92f5b78a8509up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:37cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2598912.0,"last_snapshot_bytes":237568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621849,"remote_snapshot_timestamp":1726621849,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 01:14:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34' 2024-09-18T01:14:37.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:37.729 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T01:14:37.733 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:35' 2024-09-18T01:14:37.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T01:14:37.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:37.753 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:37.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:37.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary4 2024-09-18T01:14:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:14:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:14:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary4 2024-09-18T01:14:37.778 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary590242e82-997f-4088-b460-92f5b78a8509up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:37cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2598912.0,"last_snapshot_bytes":237568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621849,"remote_snapshot_timestamp":1726621849,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 01:14:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34' 2024-09-18T01:14:37.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:37.779 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T01:14:37.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING9142614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:34cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary590242e82-997f-4088-b460-92f5b78a8509up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:37cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:36cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":2598912.0,"last_snapshot_bytes":237568,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621849,"remote_snapshot_timestamp":1726621849,"replay_state":"idle"}42614264mirror.0smithi0492024-09-18 01:14:09cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+stoppedlocal image is primary2024-09-18 01:14:34image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+unknownremote image demoted42614264mirror.0smithi0492024-09-18 01:14:35cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:34' 2024-09-18T01:14:37.784 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:37.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T01:14:37.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:35' 2024-09-18T01:14:37.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:35' 2024-09-18T01:14:37.790 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T01:14:37.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T01:14:37.794 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:37.795 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:37.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:37.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:37.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary9 2024-09-18T01:14:37.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:37.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:37.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:14:37.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:14:37.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary9 2024-09-18T01:14:37.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:37.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:37.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:37.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary8 2024-09-18T01:14:37.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:37.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:37.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:14:37.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:14:37.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary8 2024-09-18T01:14:37.959 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image demoted to non-primary 2024-09-18T01:14:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+unknown 2024-09-18T01:14:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:14:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:37.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:37.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:37.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T01:14:37.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:37.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:37.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:14:37.972 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:37.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:37.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:37.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:37.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:37.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T01:14:38.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T01:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 90242e82-997f-4088-b460-92f5b78a8509 2024-09-18T01:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:37' 2024-09-18T01:14:38.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:38.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:14:38.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:38.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T01:14:38.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:14:38.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:14:38.223 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:14:38.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:14:38.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary9 2024-09-18T01:14:38.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:14:38.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:14:38.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary8 2024-09-18T01:14:38.250 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary4 2024-09-18T01:14:38.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd2 2024-09-18T01:14:38.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:14:38.426 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd2 2024-09-18T01:14:38.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd3 2024-09-18T01:14:38.463 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:14:38.464 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd3 2024-09-18T01:14:38.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd4 2024-09-18T01:14:38.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:14:38.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd4 2024-09-18T01:14:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T01:14:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:14:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:38.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:38.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:38.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:38.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T01:14:39.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T01:14:39.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:39.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:39.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:14:39.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:39.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:39.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:39.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:39.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T01:14:39.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:39.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:39.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:14:39.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:39.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:39.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:39.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:39.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:39.072 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T01:14:39.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:39.073 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T01:14:39.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T01:14:39.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:39.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:39.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:14:39.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:39.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:39.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:39.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:39.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:39.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T01:14:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T01:14:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9e5e582f-3b8d-48a4-8031-9e27f633478c 2024-09-18T01:14:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:14:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:14:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:39.109 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:38' 2024-09-18T01:14:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 2 2024-09-18T01:14:39.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T01:14:39.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 68158950-cc3b-4cfd-bf00-d7829fce7865 2024-09-18T01:14:39.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:14:39.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:14:39.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:39.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:39.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:39.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:39.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:39.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:39.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:36' 2024-09-18T01:14:39.193 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:39.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:14:39.194 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:39.195 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T01:14:39.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T01:14:39.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 01a5a6a4-a190-4d6c-9392-bbfa30f6ec37 2024-09-18T01:14:39.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:14:39.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:14:39.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:39.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:39.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:39.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:39.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:39.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:39.200 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:36' 2024-09-18T01:14:39.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:39.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:14:39.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:39.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T01:14:39.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T01:14:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 977eb8ba-b7cf-46d7-bbaf-d2f345b9d999 2024-09-18T01:14:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:14:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:14:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:34 2024-09-18T01:14:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image demoted 2024-09-18T01:14:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:36' 2024-09-18T01:14:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 1 2024-09-18T01:14:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:39.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 4 2024-09-18T01:14:40.068 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+unknown '' '' 2024-09-18T01:14:40.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:40.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:40.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:14:40.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:40.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:40.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:40.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:40.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:40.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T01:14:40.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T01:14:40.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 90242e82-997f-4088-b460-92f5b78a8509 2024-09-18T01:14:40.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:40.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:40.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:40.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39 2024-09-18T01:14:40.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:40.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:40.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:40.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:40.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39' 2024-09-18T01:14:40.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:40.358 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:40.359 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:40.360 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:40.362 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:40.363 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:40.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:40.364 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:40.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:40.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+unknown '2024-09-18 01:14:39' 2024-09-18T01:14:40.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:40.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:40.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:14:40.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:40.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:39' 2024-09-18T01:14:40.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:40.367 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:40.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:40.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary590242e82-997f-4088-b460-92f5b78a8509up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39' 2024-09-18T01:14:40.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:40.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T01:14:40.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:40.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T01:14:40.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:40.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:40.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:40.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary5 up+unknown 2024-09-18T01:14:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:14:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:40.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:40.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:40.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:40.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:41.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+unknown '' '' 2024-09-18T01:14:41.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:41.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:41.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:14:41.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:41.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:41.119 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:41.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:41.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:41.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T01:14:41.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T01:14:41.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9e5e582f-3b8d-48a4-8031-9e27f633478c 2024-09-18T01:14:41.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:41.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:41.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:41.229 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39 2024-09-18T01:14:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39' 2024-09-18T01:14:41.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:41.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:41.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:41.235 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:41.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:41.242 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:41.243 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:41.244 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:41.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:41.245 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:41.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:41.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:41.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+unknown '2024-09-18 01:14:39' 2024-09-18T01:14:41.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:41.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:41.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:14:41.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:41.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:39' 2024-09-18T01:14:41.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:41.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:41.258 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:41.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary590242e82-997f-4088-b460-92f5b78a8509up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39' 2024-09-18T01:14:41.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:41.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T01:14:41.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:41.493 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T01:14:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:41.501 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary7 up+unknown 2024-09-18T01:14:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:14:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:41.502 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary5 up+unknown '' '' 2024-09-18T01:14:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:14:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary5 2024-09-18T01:14:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T01:14:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 90242e82-997f-4088-b460-92f5b78a8509 2024-09-18T01:14:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39 2024-09-18T01:14:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:14:41.966 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:41.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:41.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39' 2024-09-18T01:14:41.967 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:41.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:41.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:41.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:41.976 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:41.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:41.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:41.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:41.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:41.984 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:41.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:41.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:41.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary5 up+unknown '2024-09-18 01:14:39' 2024-09-18T01:14:41.992 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:41.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:41.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:14:41.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:41.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:39' 2024-09-18T01:14:41.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:41.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:41.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:42.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}2024-09-18 01:14:39image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:39image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:39image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}2024-09-18 01:14:39image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39image-primary590242e82-997f-4088-b460-92f5b78a8509up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:39image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:39image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:39image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39' 2024-09-18T01:14:42.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:42.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T01:14:42.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:42.220 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T01:14:42.226 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:42.227 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:42.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:42.232 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:42.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:42.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:42.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary5 2024-09-18T01:14:42.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:42.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:42.233 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:14:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:14:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary5 2024-09-18T01:14:42.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary7 up+unknown '' '' 2024-09-18T01:14:42.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:42.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:42.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:14:42.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:42.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:42.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:42.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:42.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:42.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary7 2024-09-18T01:14:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T01:14:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9e5e582f-3b8d-48a4-8031-9e27f633478c 2024-09-18T01:14:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:42.687 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39 2024-09-18T01:14:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:14:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39' 2024-09-18T01:14:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:42.688 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:42.691 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:42.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:42.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:42.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:42.694 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:42.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:42.697 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:42.698 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:42.699 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary7 up+unknown '2024-09-18 01:14:39' 2024-09-18T01:14:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:14:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:39' 2024-09-18T01:14:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:42.706 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:42.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}2024-09-18 01:14:39image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:39image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:39image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}2024-09-18 01:14:39image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39image-primary590242e82-997f-4088-b460-92f5b78a8509up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:39image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:39image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:39image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:39cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39' 2024-09-18T01:14:42.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:42.951 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T01:14:42.969 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:42.970 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T01:14:42.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:42.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:42.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary7 2024-09-18T01:14:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:14:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:14:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary7 2024-09-18T01:14:43.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+unknown '' '' 2024-09-18T01:14:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:14:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:43.197 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:43.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+unknown '' '' 2024-09-18T01:14:43.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:43.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:43.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:14:43.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:43.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:43.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:43.204 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:43.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:43.206 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T01:14:43.209 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:43.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T01:14:43.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+unknown '' '' 2024-09-18T01:14:43.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:43.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:43.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:14:43.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:43.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:43.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:43.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:43.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:43.215 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T01:14:43.264 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:14:43.277 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:14:43.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:14:43.302 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary5 2024-09-18T01:14:43.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:14:43.319 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary7 2024-09-18T01:14:43.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T01:14:43.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 977eb8ba-b7cf-46d7-bbaf-d2f345b9d999 2024-09-18T01:14:43.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:43.385 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39 2024-09-18T01:14:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39' 2024-09-18T01:14:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T01:14:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 01a5a6a4-a190-4d6c-9392-bbfa30f6ec37 2024-09-18T01:14:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39 2024-09-18T01:14:43.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39' 2024-09-18T01:14:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:43.391 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:43.392 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:43.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:43.394 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 68158950-cc3b-4cfd-bf00-d7829fce7865 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:39' 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:43.395 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:43.396 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:43.397 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:43.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:43.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:43.398 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:43.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:43.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:43.401 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:43.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:43.404 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:43.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:43.405 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:43.407 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:43.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:43.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:43.409 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:43.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:43.410 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+unknown '2024-09-18 01:14:39' 2024-09-18T01:14:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:14:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:39' 2024-09-18T01:14:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:43.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:43.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:43.414 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:43.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:43.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:43.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:43.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+unknown '2024-09-18 01:14:39' 2024-09-18T01:14:43.415 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:14:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:39' 2024-09-18T01:14:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+unknown '2024-09-18 01:14:39' 2024-09-18T01:14:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:14:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:14:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:39' 2024-09-18T01:14:43.418 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:43.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:43.419 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:43.692 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd5 2024-09-18T01:14:43.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd5 2024-09-18T01:14:43.693 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:14:43.702 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd6 2024-09-18T01:14:43.703 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:14:43.712 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd6 2024-09-18T01:14:43.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary590242e82-997f-4088-b460-92f5b78a8509up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39' 2024-09-18T01:14:43.764 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary590242e82-997f-4088-b460-92f5b78a8509up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39' 2024-09-18T01:14:43.768 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:43.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T01:14:43.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T01:14:43.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING5542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary590242e82-997f-4088-b460-92f5b78a8509up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+unknownremote image is not primary42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+unknownremote image is not primary2024-09-18 01:14:39image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:14:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:14:39' 2024-09-18T01:14:43.781 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T01:14:43.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T01:14:43.791 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T01:14:43.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:43.798 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:43.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:43.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:43.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:43.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:43.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary6 up+unknown 2024-09-18T01:14:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:43.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:43.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:14:43.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:43.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:43.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:43.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:43.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:43.802 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary2 up+unknown 2024-09-18T01:14:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:14:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:43.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:39' 2024-09-18T01:14:43.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T01:14:43.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:43.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:43.854 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image-primary10 up+unknown 2024-09-18T01:14:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:14:43.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:43.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:43.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:43.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:14:43.857 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:14:44.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary6 up+unknown '' '' 2024-09-18T01:14:44.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:44.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary2 up+unknown '' '' 2024-09-18T01:14:44.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:44.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:14:44.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:44.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:44.804 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:14:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary2 2024-09-18T01:14:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary6 2024-09-18T01:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster2 mirror image-primary10 up+unknown '' '' 2024-09-18T01:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:44.859 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:14:44.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:14:44.860 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:44.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:44.868 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror image status mirror/image-primary10 2024-09-18T01:14:45.016 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T01:14:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 68158950-cc3b-4cfd-bf00-d7829fce7865 2024-09-18T01:14:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:44 2024-09-18T01:14:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:14:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:44' 2024-09-18T01:14:45.017 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:45.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T01:14:45.019 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 977eb8ba-b7cf-46d7-bbaf-d2f345b9d999 2024-09-18T01:14:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:44 2024-09-18T01:14:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:14:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:44' 2024-09-18T01:14:45.020 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:45.021 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:45.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:45.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:45.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:45.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:45.032 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T01:14:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 01a5a6a4-a190-4d6c-9392-bbfa30f6ec37 2024-09-18T01:14:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:14:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:44 2024-09-18T01:14:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:14:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster1 2024-09-18T01:14:45.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+unknown 2024-09-18T01:14:45.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: remote image is not primary 2024-09-18T01:14:45.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:14:44' 2024-09-18T01:14:45.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+unknown' 2024-09-18T01:14:45.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:45.036 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:45.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:45.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:45.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:45.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:45.040 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:44' 2024-09-18T01:14:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary6 up+unknown '2024-09-18 01:14:44' 2024-09-18T01:14:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:45.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:45.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:14:45.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:45.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:44' 2024-09-18T01:14:45.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:45.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:45.047 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:45.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+unknown 2024-09-18T01:14:45.055 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:14:45.061 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: remote image is not primary 2024-09-18T01:14:45.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:14:45.062 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:45.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:14:45.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:45.065 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:45.066 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:45.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:45.067 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:45.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:44' 2024-09-18T01:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary2 up+unknown '2024-09-18 01:14:44' 2024-09-18T01:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:44' 2024-09-18T01:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:45.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:45.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:45.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:45.079 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:14:45.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:14:45.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:14:45.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:14:45.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:44' 2024-09-18T01:14:45.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image-primary10 up+unknown '2024-09-18 01:14:44' 2024-09-18T01:14:45.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:45.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:45.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:14:45.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+unknown 2024-09-18T01:14:45.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:14:44' 2024-09-18T01:14:45.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:14:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:14:45.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2024-09-18T01:14:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}2024-09-18 01:14:39image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:44image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:44image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}2024-09-18 01:14:39image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39image-primary590242e82-997f-4088-b460-92f5b78a8509up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}2024-09-18 01:14:44image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:44image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}2024-09-18 01:14:44image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39' 2024-09-18T01:14:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:45.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T01:14:45.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:44' 2024-09-18T01:14:45.450 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T01:14:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:45.461 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:45.482 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:45.486 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:45.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:45.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:45.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary6 2024-09-18T01:14:45.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:45.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:45.487 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:14:45.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:14:45.488 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary6 2024-09-18T01:14:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}2024-09-18 01:14:39image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:44image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:44image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}2024-09-18 01:14:39image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39image-primary590242e82-997f-4088-b460-92f5b78a8509up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}2024-09-18 01:14:44image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:44image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}2024-09-18 01:14:44image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39' 2024-09-18T01:14:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T01:14:45.535 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:44' 2024-09-18T01:14:45.536 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T01:14:45.546 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:45.552 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:45.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:45.560 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:45.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary2 2024-09-18T01:14:45.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:45.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:45.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:14:45.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:14:45.564 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary2 2024-09-18T01:14:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='WARNINGOKWARNING3742614264mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}2024-09-18 01:14:39image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:44image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:44image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}2024-09-18 01:14:39image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39image-primary590242e82-997f-4088-b460-92f5b78a8509up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}2024-09-18 01:14:44image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+unknownremote image is not primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+unknownremote image is not primary2024-09-18 01:14:44image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}2024-09-18 01:14:44image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+stoppedlocal image is primary42614264mirror.0smithi0492024-09-18 01:14:44cluster1c0705d42-6080-4a61-ba3e-72d757843e94up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}2024-09-18 01:14:39' 2024-09-18T01:14:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:14:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T01:14:45.636 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:14:44' 2024-09-18T01:14:45.652 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T01:14:45.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+unknown 2024-09-18T01:14:45.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+unknown 2024-09-18T01:14:45.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+unknown 2024-09-18T01:14:45.659 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+unknown 2024-09-18T01:14:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:14:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_image cluster2 mirror image-primary10 2024-09-18T01:14:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:14:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:14:45.660 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:14:45.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local force= 2024-09-18T01:14:45.661 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/image-primary10 2024-09-18T01:14:46.249 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:14:46.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:14:46.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:Image promoted to primary 2024-09-18T01:14:46.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:14:46.283 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary2 2024-09-18T01:14:46.288 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:14:46.292 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary10 2024-09-18T01:14:46.293 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ nbd == \n\b\d ]] 2024-09-18T01:14:46.296 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo rbd --cluster cluster2 device map -t nbd -o try-netlink mirror/image-primary6 2024-09-18T01:14:46.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd8 2024-09-18T01:14:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd7 2024-09-18T01:14:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd7 2024-09-18T01:14:46.651 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:14:46.655 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:14:46.656 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd8 2024-09-18T01:14:46.673 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ dev=/dev/nbd9 2024-09-18T01:14:46.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sudo md5sum /dev/nbd9 2024-09-18T01:14:46.674 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ awk '{print $1}' 2024-09-18T01:16:14.333 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:16:14.307+0000 7fd4c158b640 -1 reset not still connected to 0x5643587075f0 2024-09-18T01:16:14.333 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:16:14.315+0000 7fd4c158b640 -1 reset not still connected to 0x564358707d40 2024-09-18T01:16:14.333 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:16:14.315+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a340 2024-09-18T01:16:14.333 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:16:14.315+0000 7fd4c158b640 -1 reset not still connected to 0x56435881a410 2024-09-18T01:16:14.333 INFO:tasks.ceph.cluster2.osd.2.smithi049.stderr:2024-09-18T01:16:14.315+0000 7fd4c158b640 -1 reset not still connected to 0x56435881b110 2024-09-18T01:16:25.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=88340d70a35dd3634e58c9ad1214effc 2024-09-18T01:16:25.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd0 2024-09-18T01:16:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=77c91e0d9e3cc0edd3e96d2cc9698045 2024-09-18T01:16:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd1 2024-09-18T01:16:26.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=06c7e007505e08ac519a8e9e1072cfb9 2024-09-18T01:16:26.939 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd2 2024-09-18T01:16:27.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 88340d70a35dd3634e58c9ad1214effc != \8\8\3\4\0\d\7\0\a\3\5\d\d\3\6\3\4\e\5\8\c\9\a\d\1\2\1\4\e\f\f\c ]] 2024-09-18T01:16:27.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:16:27.071 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 138423 2024-09-18T01:16:27.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 77c91e0d9e3cc0edd3e96d2cc9698045 != \7\7\c\9\1\e\0\d\9\e\3\c\c\0\e\d\d\3\e\9\6\d\2\c\c\9\6\9\8\0\4\5 ]] 2024-09-18T01:16:27.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=1d7b78367e6688efc34bce166fc6879a 2024-09-18T01:16:27.965 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd4 2024-09-18T01:16:28.081 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 06c7e007505e08ac519a8e9e1072cfb9 != \0\6\c\7\e\0\0\7\5\0\5\e\0\8\a\c\5\1\9\a\8\e\9\e\1\0\7\2\c\f\b\9 ]] 2024-09-18T01:16:28.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=b0f32cd4bff1e0b1e816bb401888bd37 2024-09-18T01:16:28.477 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd3 2024-09-18T01:16:28.684 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 1d7b78367e6688efc34bce166fc6879a != \1\d\7\b\7\8\3\6\7\e\6\6\8\8\e\f\c\3\4\b\c\e\1\6\6\f\c\6\8\7\9\a ]] 2024-09-18T01:16:29.301 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ b0f32cd4bff1e0b1e816bb401888bd37 != \b\0\f\3\2\c\d\4\b\f\f\1\e\0\b\1\e\8\1\6\b\b\4\0\1\8\8\8\b\d\3\7 ]] 2024-09-18T01:16:32.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=5c17a00ccaded73adbf2ff3f23d90d13 2024-09-18T01:16:32.792 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd6 2024-09-18T01:16:32.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=7b3d95e809aa30394eb8afe2b21aa632 2024-09-18T01:16:32.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd5 2024-09-18T01:16:33.642 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 7b3d95e809aa30394eb8afe2b21aa632 != \7\b\3\d\9\5\e\8\0\9\a\a\3\0\3\9\4\e\b\8\a\f\e\2\b\2\1\a\a\6\3\2 ]] 2024-09-18T01:16:33.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 5c17a00ccaded73adbf2ff3f23d90d13 != \5\c\1\7\a\0\0\c\c\a\d\e\d\7\3\a\d\b\f\2\f\f\3\f\2\3\d\9\0\d\1\3 ]] 2024-09-18T01:16:34.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=60428a142d540eee7ad53d4e4cb767e7 2024-09-18T01:16:34.043 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd8 2024-09-18T01:16:34.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=7d0fe805a523bb4cddbac8609b410d87 2024-09-18T01:16:34.216 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd7 2024-09-18T01:16:34.298 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ promote_md5=0d1978ecf45d9f238aaf370a281abcfb 2024-09-18T01:16:34.299 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sudo rbd --cluster cluster2 device unmap -t nbd /dev/nbd9 2024-09-18T01:16:34.627 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 60428a142d540eee7ad53d4e4cb767e7 != \6\0\4\2\8\a\1\4\2\d\5\4\0\e\e\e\7\a\d\5\3\d\4\e\4\c\b\7\6\7\e\7 ]] 2024-09-18T01:16:34.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 7d0fe805a523bb4cddbac8609b410d87 != \7\d\0\f\e\8\0\5\a\5\2\3\b\b\4\c\d\d\b\a\c\8\6\0\9\b\4\1\0\d\8\7 ]] 2024-09-18T01:16:34.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:16:34.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 138424 2024-09-18T01:16:34.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:16:34.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 138426 2024-09-18T01:16:34.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:16:34.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 138429 2024-09-18T01:16:34.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:16:34.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 138431 2024-09-18T01:16:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ [[ 0d1978ecf45d9f238aaf370a281abcfb != \0\d\1\9\7\8\e\c\f\4\5\d\9\f\2\3\8\a\a\f\3\7\0\a\2\8\1\a\b\c\f\b ]] 2024-09-18T01:16:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:16:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 138433 2024-09-18T01:16:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:16:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 138434 2024-09-18T01:16:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:16:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 138437 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for pid in ${COMPARE_PIDS[@]} 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait 138438 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ (( RET != 0 )) 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary1 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary1 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary1 up+replaying 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:16:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:34.823 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:16:35.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary1 up+replaying '' '' 2024-09-18T01:16:35.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:35.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:35.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:16:35.821 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:35.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:35.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:35.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:35.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:35.822 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary1 2024-09-18T01:16:35.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary1: 2024-09-18T01:16:35.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: d0f03b0b-70ae-444b-9369-278125560577 2024-09-18T01:16:35.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:16:35.912 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70586368.0,"last_snapshot_bytes":70586368,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622077,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"} 2024-09-18T01:16:35.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:16:35.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:09 2024-09-18T01:16:35.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:16:35.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:16:35.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:16:35.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:16:35.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:34' 2024-09-18T01:16:35.913 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:16:35.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:16:35.945 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:16:35.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70586368.0,"last_snapshot_bytes":70586368,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622077,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"} 2024-09-18T01:16:35.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:16:35.948 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:35.949 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:16:35.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:35.950 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:16:35.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:16:35.952 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:16:35.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:16:36.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:09' 2024-09-18T01:16:36.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary1 up+replaying '2024-09-18 01:16:09' 2024-09-18T01:16:36.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:36.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:36.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:16:36.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:36.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:16:09' 2024-09-18T01:16:36.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:36.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:36.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:16:36.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK1042544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1d0f03b0b-70ae-444b-9369-278125560577up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70586368.0,"last_snapshot_bytes":70586368,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622077,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:34image-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72318976.0,"last_snapshot_bytes":72318976,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:34image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70131712.0,"last_snapshot_bytes":70131712,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:34image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70119424.0,"last_snapshot_bytes":70119424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622077,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:34image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72093696.0,"last_snapshot_bytes":72093696,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:34image-primary590242e82-997f-4088-b460-92f5b78a8509up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":115814400.0,"last_snapshot_bytes":115814400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:34image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:34image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62263296.0,"last_snapshot_bytes":62263296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:34image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72396800.0,"last_snapshot_bytes":72396800,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:34image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72249344.0,"last_snapshot_bytes":72249344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:09cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:34' 2024-09-18T01:16:36.202 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:16:36.203 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/last_update' 2024-09-18T01:16:36.973 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:09' 2024-09-18T01:16:36.974 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary1'\'']/state' 2024-09-18T01:16:36.978 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:16:36.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:36.979 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:16:36.980 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:36.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:36.981 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:36.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary1 2024-09-18T01:16:36.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:16:36.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:36.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:16:36.982 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary1 2024-09-18T01:16:37.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary1 2024-09-18T01:16:38.704 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:16:38.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary1 2024-09-18T01:16:38.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:38.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:38.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary1 2024-09-18T01:16:38.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:38.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:16:38.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary1 2024-09-18T01:16:38.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:38.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary1 2024-09-18T01:16:38.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary2 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary2 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary2 up+replaying 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:38.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:38.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:38.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:16:39.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary2 up+replaying '' '' 2024-09-18T01:16:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:16:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:39.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary2 2024-09-18T01:16:40.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary2: 2024-09-18T01:16:40.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 68158950-cc3b-4cfd-bf00-d7829fce7865 2024-09-18T01:16:40.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:16:40.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70131712.0,"last_snapshot_bytes":70131712,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"} 2024-09-18T01:16:40.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:16:40.033 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:39 2024-09-18T01:16:40.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:16:40.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:16:40.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:16:40.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:16:40.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:39' 2024-09-18T01:16:40.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:16:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:16:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:16:40.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70131712.0,"last_snapshot_bytes":70131712,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"} 2024-09-18T01:16:40.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:16:40.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:40.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:16:40.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:40.039 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:16:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:16:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:16:40.041 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:16:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:16:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary2 up+replaying '2024-09-18 01:16:39' 2024-09-18T01:16:40.044 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:40.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:40.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:16:40.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:40.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:16:39' 2024-09-18T01:16:40.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:40.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:40.045 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:16:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK942544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72318976.0,"last_snapshot_bytes":72318976,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary268158950-cc3b-4cfd-bf00-d7829fce7865up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70131712.0,"last_snapshot_bytes":70131712,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70119424.0,"last_snapshot_bytes":70119424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622077,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72093696.0,"last_snapshot_bytes":72093696,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary590242e82-997f-4088-b460-92f5b78a8509up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":115814400.0,"last_snapshot_bytes":115814400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62263296.0,"last_snapshot_bytes":62263296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72396800.0,"last_snapshot_bytes":72396800,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72249344.0,"last_snapshot_bytes":72249344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39' 2024-09-18T01:16:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:16:40.201 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/last_update' 2024-09-18T01:16:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:16:40.207 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary2'\'']/state' 2024-09-18T01:16:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:16:40.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:40.212 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:16:40.213 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary2 2024-09-18T01:16:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:16:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:16:40.214 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary2 2024-09-18T01:16:40.278 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary2 2024-09-18T01:16:41.132 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:16:41.129+0000 7f849c00d640 -1 reset not still connected to 0x5629a8692c30 2024-09-18T01:16:41.143 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:16:41.137+0000 7f849c00d640 -1 reset not still connected to 0x5629addd9c70 2024-09-18T01:16:41.144 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:16:41.137+0000 7f849c00d640 -1 reset not still connected to 0x5629b72c3860 2024-09-18T01:16:41.144 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:16:41.137+0000 7f849c00d640 -1 reset not still connected to 0x5629bd82ba00 2024-09-18T01:16:41.144 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:16:41.137+0000 7f849c00d640 -1 reset not still connected to 0x5629c4538dd0 2024-09-18T01:16:41.144 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:16:41.137+0000 7f849c00d640 -1 reset not still connected to 0x5629d5370680 2024-09-18T01:16:41.144 INFO:tasks.ceph.cluster1.osd.0.smithi028.stderr:2024-09-18T01:16:41.137+0000 7f849c00d640 -1 reset not still connected to 0x562a1381ec30 2024-09-18T01:16:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:16:42.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary2 2024-09-18T01:16:42.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:42.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:42.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary2 2024-09-18T01:16:42.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:42.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:16:42.695 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary2 2024-09-18T01:16:42.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:42.734 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary2 2024-09-18T01:16:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:16:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary3 2024-09-18T01:16:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary3 2024-09-18T01:16:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:16:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:16:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary3 up+replaying 2024-09-18T01:16:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:16:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:42.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:16:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary3 up+replaying '' '' 2024-09-18T01:16:43.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:43.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:43.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:16:43.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:43.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:43.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:43.738 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:43.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:43.739 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary3 2024-09-18T01:16:43.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary3: 2024-09-18T01:16:43.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 821edf46-8d24-4ea2-bb84-66b4b58ea23a 2024-09-18T01:16:43.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:16:43.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70119424.0,"last_snapshot_bytes":70119424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622077,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"} 2024-09-18T01:16:43.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:16:43.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:39 2024-09-18T01:16:43.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:16:43.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:16:43.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:16:43.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:16:43.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:39' 2024-09-18T01:16:43.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:16:43.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:16:43.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:16:43.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70119424.0,"last_snapshot_bytes":70119424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622077,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"} 2024-09-18T01:16:43.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:16:43.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:43.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:16:43.831 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:43.832 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:16:43.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:16:43.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:16:43.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:16:43.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:16:43.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary3 up+replaying '2024-09-18 01:16:39' 2024-09-18T01:16:43.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:43.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:43.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:16:43.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:43.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:16:39' 2024-09-18T01:16:43.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:43.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:43.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:16:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK842544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72318976.0,"last_snapshot_bytes":72318976,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary3821edf46-8d24-4ea2-bb84-66b4b58ea23aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":70119424.0,"last_snapshot_bytes":70119424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622077,"remote_snapshot_timestamp":1726622077,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72093696.0,"last_snapshot_bytes":72093696,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary590242e82-997f-4088-b460-92f5b78a8509up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":115814400.0,"last_snapshot_bytes":115814400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62263296.0,"last_snapshot_bytes":62263296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72396800.0,"last_snapshot_bytes":72396800,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72249344.0,"last_snapshot_bytes":72249344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:39' 2024-09-18T01:16:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:16:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/last_update' 2024-09-18T01:16:44.191 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:16:44.192 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary3'\'']/state' 2024-09-18T01:16:44.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:16:44.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:44.196 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:16:44.198 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:44.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:44.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:44.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary3 2024-09-18T01:16:44.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:16:44.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:44.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:16:44.199 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary3 2024-09-18T01:16:44.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary3 2024-09-18T01:16:46.205 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:16:46.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary3 2024-09-18T01:16:46.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:46.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:46.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary3 2024-09-18T01:16:46.210 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:46.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:16:46.211 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary3 2024-09-18T01:16:46.252 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:46.253 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary3 2024-09-18T01:16:46.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:16:46.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary4 2024-09-18T01:16:46.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary4 2024-09-18T01:16:46.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:46.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:46.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:16:46.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:16:46.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary4 up+replaying 2024-09-18T01:16:46.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:46.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:46.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:16:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:16:47.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary4 up+replaying '' '' 2024-09-18T01:16:47.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:47.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:47.255 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:16:47.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:47.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:47.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:47.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:47.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:47.256 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary4 2024-09-18T01:16:47.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary4: 2024-09-18T01:16:47.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: ee4c71cc-a845-47e1-b515-dffaef47a382 2024-09-18T01:16:47.345 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:16:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72093696.0,"last_snapshot_bytes":72093696,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"} 2024-09-18T01:16:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:16:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:39 2024-09-18T01:16:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:16:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:16:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:16:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:16:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:44' 2024-09-18T01:16:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:16:47.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:16:47.347 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:16:47.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72093696.0,"last_snapshot_bytes":72093696,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"} 2024-09-18T01:16:47.348 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:16:47.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:47.349 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:16:47.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:47.351 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:16:47.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:16:47.352 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:16:47.353 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:16:47.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:16:47.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary4 up+replaying '2024-09-18 01:16:39' 2024-09-18T01:16:47.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:47.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:47.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:16:47.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:47.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:16:39' 2024-09-18T01:16:47.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:47.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:47.357 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:16:47.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK742544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72318976.0,"last_snapshot_bytes":72318976,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:44image-primary4ee4c71cc-a845-47e1-b515-dffaef47a382up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72093696.0,"last_snapshot_bytes":72093696,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:44image-primary590242e82-997f-4088-b460-92f5b78a8509up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":115814400.0,"last_snapshot_bytes":115814400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:44image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:44image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62263296.0,"last_snapshot_bytes":62263296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:44image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72396800.0,"last_snapshot_bytes":72396800,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:44image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72249344.0,"last_snapshot_bytes":72249344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:44' 2024-09-18T01:16:47.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:16:47.492 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/last_update' 2024-09-18T01:16:47.497 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:16:47.498 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary4'\'']/state' 2024-09-18T01:16:47.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:16:47.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:47.503 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:16:47.504 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:47.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:47.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:47.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary4 2024-09-18T01:16:47.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:16:47.505 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:47.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:16:47.506 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary4 2024-09-18T01:16:47.565 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary4 2024-09-18T01:16:49.705 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:16:49.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary4 2024-09-18T01:16:49.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:49.709 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:49.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary4 2024-09-18T01:16:49.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:49.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:16:49.710 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary4 2024-09-18T01:16:49.746 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:49.747 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary4 2024-09-18T01:16:49.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:16:49.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary5 2024-09-18T01:16:49.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary5 2024-09-18T01:16:49.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:49.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:49.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:16:49.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:16:49.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary5 up+replaying 2024-09-18T01:16:49.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:49.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:49.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:16:49.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:49.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:49.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:49.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:49.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:16:50.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary5 up+replaying '' '' 2024-09-18T01:16:50.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:50.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:50.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:16:50.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:50.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:50.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:50.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:50.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:50.750 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary5 2024-09-18T01:16:50.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary5: 2024-09-18T01:16:50.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 90242e82-997f-4088-b460-92f5b78a8509 2024-09-18T01:16:50.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:16:50.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":115814400.0,"last_snapshot_bytes":115814400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"} 2024-09-18T01:16:50.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:16:50.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:39 2024-09-18T01:16:50.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:16:50.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:16:50.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:16:50.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:16:50.833 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:49' 2024-09-18T01:16:50.834 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:16:50.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:16:50.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:16:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":115814400.0,"last_snapshot_bytes":115814400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"} 2024-09-18T01:16:50.836 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:16:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:16:50.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:50.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:16:50.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:16:50.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:16:50.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:16:50.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:16:50.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary5 up+replaying '2024-09-18 01:16:39' 2024-09-18T01:16:50.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:50.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:50.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:16:50.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:50.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:16:39' 2024-09-18T01:16:50.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:50.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:50.843 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:16:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK642544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72318976.0,"last_snapshot_bytes":72318976,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:49image-primary590242e82-997f-4088-b460-92f5b78a8509up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":115814400.0,"last_snapshot_bytes":115814400,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:49image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:49image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62263296.0,"last_snapshot_bytes":62263296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:49image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72396800.0,"last_snapshot_bytes":72396800,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:49image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72249344.0,"last_snapshot_bytes":72249344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:49' 2024-09-18T01:16:51.012 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:16:51.013 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/last_update' 2024-09-18T01:16:51.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:16:51.018 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary5'\'']/state' 2024-09-18T01:16:51.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:16:51.023 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:51.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:16:51.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:51.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:51.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:51.025 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary5 2024-09-18T01:16:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:16:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:16:51.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary5 2024-09-18T01:16:51.082 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary5 2024-09-18T01:16:52.725 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:16:52.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary5 2024-09-18T01:16:52.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:52.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:52.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary5 2024-09-18T01:16:52.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:52.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:16:52.730 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary5 2024-09-18T01:16:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:52.769 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary5 2024-09-18T01:16:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:16:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary6 2024-09-18T01:16:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary6 2024-09-18T01:16:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:16:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:16:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary6 up+replaying 2024-09-18T01:16:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:52.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:52.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:16:52.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:52.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:52.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:52.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:52.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:16:53.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary6 up+replaying '' '' 2024-09-18T01:16:53.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:53.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:53.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:16:53.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:53.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:53.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:53.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:53.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:53.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary6 2024-09-18T01:16:53.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary6: 2024-09-18T01:16:53.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 977eb8ba-b7cf-46d7-bbaf-d2f345b9d999 2024-09-18T01:16:53.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:16:53.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"} 2024-09-18T01:16:53.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:16:53.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:39 2024-09-18T01:16:53.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:16:53.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:16:53.883 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:16:53.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:16:53.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:49' 2024-09-18T01:16:53.884 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:16:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:16:53.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:16:53.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"} 2024-09-18T01:16:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:16:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:53.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:16:53.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:53.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:16:53.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:16:53.892 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:16:53.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:16:53.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:16:53.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary6 up+replaying '2024-09-18 01:16:39' 2024-09-18T01:16:53.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:53.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:53.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:16:53.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:53.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:16:39' 2024-09-18T01:16:53.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:53.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:53.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:16:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK542544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72318976.0,"last_snapshot_bytes":72318976,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:49image-primary6977eb8ba-b7cf-46d7-bbaf-d2f345b9d999up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72376320.0,"last_snapshot_bytes":72376320,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:49image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62263296.0,"last_snapshot_bytes":62263296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:49image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72396800.0,"last_snapshot_bytes":72396800,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:49image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72249344.0,"last_snapshot_bytes":72249344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:49' 2024-09-18T01:16:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:16:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/last_update' 2024-09-18T01:16:54.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:16:54.030 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary6'\'']/state' 2024-09-18T01:16:54.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:16:54.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:54.034 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:16:54.035 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:54.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:54.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:54.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary6 2024-09-18T01:16:54.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:16:54.037 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:54.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:16:54.038 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary6 2024-09-18T01:16:54.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary6 2024-09-18T01:16:54.791 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:16:54.785+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef1c70 2024-09-18T01:16:54.792 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:16:54.789+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4000 2024-09-18T01:16:54.792 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:16:54.789+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4750 2024-09-18T01:16:54.792 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:16:54.789+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6734b1ee0 2024-09-18T01:16:54.792 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:16:54.789+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d67e98f1e0 2024-09-18T01:16:55.752 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:16:55.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary6 2024-09-18T01:16:55.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:55.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:55.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary6 2024-09-18T01:16:55.757 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:55.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:16:55.758 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary6 2024-09-18T01:16:55.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:55.797 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary6 2024-09-18T01:16:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:16:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary7 2024-09-18T01:16:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary7 2024-09-18T01:16:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:16:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:16:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary7 up+replaying 2024-09-18T01:16:55.799 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:55.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:55.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:16:55.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:55.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:55.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:55.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:55.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:16:56.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary7 up+replaying '' '' 2024-09-18T01:16:56.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:56.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:56.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:16:56.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:56.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:56.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:56.800 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:56.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:56.801 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary7 2024-09-18T01:16:56.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary7: 2024-09-18T01:16:56.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 9e5e582f-3b8d-48a4-8031-9e27f633478c 2024-09-18T01:16:56.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:16:56.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62263296.0,"last_snapshot_bytes":62263296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"} 2024-09-18T01:16:56.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:16:56.885 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:39 2024-09-18T01:16:56.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:16:56.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:16:56.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:16:56.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:16:56.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:54' 2024-09-18T01:16:56.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:16:56.886 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:16:56.887 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:16:56.888 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62263296.0,"last_snapshot_bytes":62263296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"} 2024-09-18T01:16:56.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:16:56.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:56.889 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:16:56.890 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:56.891 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:16:56.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:16:56.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:16:56.893 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:16:56.895 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:16:56.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary7 up+replaying '2024-09-18 01:16:39' 2024-09-18T01:16:56.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:56.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:56.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:16:56.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:56.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:16:39' 2024-09-18T01:16:56.896 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:16:56.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:16:56.897 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:16:57.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK442544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72318976.0,"last_snapshot_bytes":72318976,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:54image-primary79e5e582f-3b8d-48a4-8031-9e27f633478cup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":62263296.0,"last_snapshot_bytes":62263296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622083,"remote_snapshot_timestamp":1726622083,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:54image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72396800.0,"last_snapshot_bytes":72396800,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:54image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72249344.0,"last_snapshot_bytes":72249344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:54' 2024-09-18T01:16:57.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:16:57.064 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/last_update' 2024-09-18T01:16:57.069 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:16:57.070 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary7'\'']/state' 2024-09-18T01:16:57.074 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:16:57.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:16:57.075 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:16:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:16:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary7 2024-09-18T01:16:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:16:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:16:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary7 2024-09-18T01:16:57.137 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary7 2024-09-18T01:16:59.225 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:16:59.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary7 2024-09-18T01:16:59.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:59.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:59.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary7 2024-09-18T01:16:59.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:59.230 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:16:59.231 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary7 2024-09-18T01:16:59.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:16:59.269 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary7 2024-09-18T01:16:59.270 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:16:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary8 2024-09-18T01:16:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary8 2024-09-18T01:16:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:16:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:16:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary8 up+replaying 2024-09-18T01:16:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:16:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:16:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:16:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:16:59.271 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:16:59.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:16:59.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:16:59.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:17:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary8 up+replaying '' '' 2024-09-18T01:17:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:17:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:17:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:17:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:17:00.272 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:17:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:17:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary8 2024-09-18T01:17:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary8: 2024-09-18T01:17:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 8c413c2e-0975-4386-8c15-1b1c7c115008 2024-09-18T01:17:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:17:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72396800.0,"last_snapshot_bytes":72396800,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"} 2024-09-18T01:17:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:17:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:39 2024-09-18T01:17:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:17:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:17:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:17:00.365 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:17:00.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:59' 2024-09-18T01:17:00.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:17:00.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:17:00.366 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:17:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72396800.0,"last_snapshot_bytes":72396800,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"} 2024-09-18T01:17:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:17:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:17:00.369 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:17:00.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:17:00.370 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:17:00.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:17:00.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:17:00.372 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:17:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:17:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary8 up+replaying '2024-09-18 01:16:39' 2024-09-18T01:17:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:17:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:17:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:16:39' 2024-09-18T01:17:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:17:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:17:00.375 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:17:00.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK342544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72318976.0,"last_snapshot_bytes":72318976,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:59image-primary88c413c2e-0975-4386-8c15-1b1c7c115008up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72396800.0,"last_snapshot_bytes":72396800,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:59image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72249344.0,"last_snapshot_bytes":72249344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:59' 2024-09-18T01:17:00.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:17:00.465 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/last_update' 2024-09-18T01:17:00.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:17:00.469 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary8'\'']/state' 2024-09-18T01:17:00.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:17:00.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:17:00.473 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:17:00.474 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:17:00.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:17:00.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:17:00.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary8 2024-09-18T01:17:00.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:17:00.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:00.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:17:00.475 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary8 2024-09-18T01:17:00.528 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary8 2024-09-18T01:17:02.780 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:17:02.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary8 2024-09-18T01:17:02.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:02.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:02.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary8 2024-09-18T01:17:02.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:17:02.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:17:02.785 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary8 2024-09-18T01:17:02.825 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:17:02.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary8 2024-09-18T01:17:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:17:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary9 2024-09-18T01:17:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary9 2024-09-18T01:17:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:17:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:17:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary9 up+replaying 2024-09-18T01:17:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:02.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:17:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:17:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:17:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:17:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:17:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:17:03.774 INFO:tasks.ceph.cluster1.osd.1.smithi028.stderr:2024-09-18T01:17:03.768+0000 7fac32fac640 -1 reset not still connected to 0x55a83c810270 2024-09-18T01:17:03.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary9 up+replaying '' '' 2024-09-18T01:17:03.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:03.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:17:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:17:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:17:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:17:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:17:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:17:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary9 2024-09-18T01:17:03.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary9: 2024-09-18T01:17:03.953 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: b5da1212-bf20-4dc9-bf59-f911a78837d4 2024-09-18T01:17:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:17:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72249344.0,"last_snapshot_bytes":72249344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"} 2024-09-18T01:17:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:17:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:39 2024-09-18T01:17:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:17:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:17:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:17:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:17:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:59' 2024-09-18T01:17:03.954 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:17:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:17:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:17:03.955 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72249344.0,"last_snapshot_bytes":72249344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"} 2024-09-18T01:17:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:17:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:17:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:17:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:17:03.956 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:17:03.957 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary9 up+replaying '2024-09-18 01:16:39' 2024-09-18T01:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:16:39' 2024-09-18T01:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:17:03.958 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:17:04.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK242544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72318976.0,"last_snapshot_bytes":72318976,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:59image-primary9b5da1212-bf20-4dc9-bf59-f911a78837d4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72249344.0,"last_snapshot_bytes":72249344,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622078,"remote_snapshot_timestamp":1726622078,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:16:59' 2024-09-18T01:17:04.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:17:04.077 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/last_update' 2024-09-18T01:17:04.084 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:17:04.085 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary9'\'']/state' 2024-09-18T01:17:04.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:17:04.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:17:04.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:17:04.088 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:17:04.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:17:04.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:17:04.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary9 2024-09-18T01:17:04.089 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:17:04.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:04.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:17:04.090 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary9 2024-09-18T01:17:04.149 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary9 2024-09-18T01:17:05.782 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary9 2024-09-18T01:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary9 2024-09-18T01:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:17:05.787 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary9 2024-09-18T01:17:05.826 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:17:05.827 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary9 2024-09-18T01:17:05.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for j in {1..10} 2024-09-18T01:17:05.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ IMG=image-primary10 2024-09-18T01:17:05.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror image-primary10 2024-09-18T01:17:05.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:05.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:05.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:17:05.828 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:17:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_status_in_pool_dir cluster1 mirror image-primary10 up+replaying 2024-09-18T01:17:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:17:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:17:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:17:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:17:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:17:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:17:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_status_in_pool_dir cluster1 mirror image-primary10 up+replaying '' '' 2024-09-18T01:17:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:17:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:17:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local description_pattern= 2024-09-18T01:17:06.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local service_pattern= 2024-09-18T01:17:06.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:17:06.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:17:06.830 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror image status mirror/image-primary10 2024-09-18T01:17:06.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='image-primary10: 2024-09-18T01:17:06.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: global_id: 01a5a6a4-a190-4d6c-9392-bbfa30f6ec37 2024-09-18T01:17:06.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+replaying 2024-09-18T01:17:06.924 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72318976.0,"last_snapshot_bytes":72318976,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"} 2024-09-18T01:17:06.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: service: mirror.0 on smithi049 2024-09-18T01:17:06.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:16:39 2024-09-18T01:17:06.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: peer_sites: 2024-09-18T01:17:06.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: name: cluster2 2024-09-18T01:17:06.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: state: up+stopped 2024-09-18T01:17:06.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: description: local image is primary 2024-09-18T01:17:06.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: last_update: 2024-09-18 01:17:04' 2024-09-18T01:17:06.925 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:17:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: state: up+replaying 2024-09-18T01:17:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^ description: .*' 2024-09-18T01:17:06.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72318976.0,"last_snapshot_bytes":72318976,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"} 2024-09-18T01:17:06.927 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n '' ']' 2024-09-18T01:17:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:17:06.928 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep '^up+' 2024-09-18T01:17:06.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:17:06.929 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:17:06.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout: service: mirror.0 on smithi049 2024-09-18T01:17:06.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update 2024-09-18T01:17:06.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:17:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:17:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test_mirror_pool_status_verbose cluster1 mirror image-primary10 up+replaying '2024-09-18 01:16:39' 2024-09-18T01:17:06.933 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:17:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local state_pattern=up+replaying 2024-09-18T01:17:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local 'prev_last_update=2024-09-18 01:16:39' 2024-09-18T01:17:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local status 2024-09-18T01:17:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ CEPH_ARGS= 2024-09-18T01:17:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:17:07.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ status='OKOKOK142544257mirror.0smithi04919.1.1-330-gc2ddf972trueOKimage-primary1001a5a6a4-a190-4d6c-9392-bbfa30f6ec37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":72318976.0,"last_snapshot_bytes":72318976,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726622086,"remote_snapshot_timestamp":1726622086,"replay_state":"idle"}42544257mirror.0smithi0492024-09-18 01:16:39cluster22637a684-d2d5-40a5-af2a-7a7315c74555up+stoppedlocal image is primary2024-09-18 01:17:04' 2024-09-18T01:17:07.086 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local last_update state 2024-09-18T01:17:07.087 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/last_update' 2024-09-18T01:17:07.091 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ last_update='2024-09-18 01:16:39' 2024-09-18T01:17:07.092 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image-primary10'\'']/state' 2024-09-18T01:17:07.096 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ state=up+replaying 2024-09-18T01:17:07.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo up+replaying 2024-09-18T01:17:07.097 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep up+replaying 2024-09-18T01:17:07.098 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:up+replaying 2024-09-18T01:17:07.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:17:07.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:17:07.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ remove_image cluster2 mirror image-primary10 2024-09-18T01:17:07.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:17:07.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:07.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:17:07.099 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 snap purge mirror/image-primary10 2024-09-18T01:17:07.161 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster=cluster2 rm mirror/image-primary10 2024-09-18T01:17:08.251 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:17:08.248+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4000 2024-09-18T01:17:08.252 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:17:08.248+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d672ef4750 2024-09-18T01:17:08.252 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:17:08.248+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6734b1ee0 2024-09-18T01:17:08.252 INFO:tasks.ceph.cluster1.osd.2.smithi028.stderr:2024-09-18T01:17:08.248+0000 7fb0cf6d6640 -1 reset not still connected to 0x55d6735afee0 2024-09-18T01:17:09.337 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:17:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ wait_for_image_removal cluster1 mirror image-primary10 2024-09-18T01:17:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pool=mirror 2024-09-18T01:17:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local image=image-primary10 2024-09-18T01:17:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:17:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rbd --cluster cluster1 ls mirror 2024-09-18T01:17:09.342 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ grep -wq image-primary10 2024-09-18T01:17:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ return 0 2024-09-18T01:17:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -rf test-primary10 2024-09-18T01:17:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:OK 2024-09-18T01:17:09.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo OK 2024-09-18T01:17:09.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ cleanup 0 2024-09-18T01:17:09.388 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local error_code=0 2024-09-18T01:17:09.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ set +e 2024-09-18T01:17:09.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' 0 -ne 0 ']' 2024-09-18T01:17:09.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -z '' ']' 2024-09-18T01:17:09.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster instance 2024-09-18T01:17:09.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-18T01:17:09.389 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2024-09-18T01:17:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:pool 'mirror' does not exist 2024-09-18T01:17:10.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-18T01:17:10.138 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2024-09-18T01:17:10.996 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:pool 'mirror' does not exist 2024-09-18T01:17:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-18T01:17:11.024 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2024-09-18T01:17:12.136 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:pool 'mirror_parent' does not exist 2024-09-18T01:17:12.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ CEPH_ARGS= 2024-09-18T01:17:12.155 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2024-09-18T01:17:13.558 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:pool 'mirror_parent' does not exist 2024-09-18T01:17:13.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2024-09-18T01:17:13.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ stop_mirrors cluster1 2024-09-18T01:17:13.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1 2024-09-18T01:17:13.576 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local sig= 2024-09-18T01:17:13.577 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ seq 0 0 2024-09-18T01:17:13.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:17:13.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ stop_mirror cluster1:0 '' 2024-09-18T01:17:13.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster1:0 2024-09-18T01:17:13.589 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local sig= 2024-09-18T01:17:13.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test -n '' 2024-09-18T01:17:13.590 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pid 2024-09-18T01:17:13.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ daemon_pid_file cluster1:0 2024-09-18T01:17:13.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local cluster=cluster1:0 2024-09-18T01:17:13.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local instance 2024-09-18T01:17:13.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:17:13.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local val=cluster1:0 2024-09-18T01:17:13.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local cluster_var_name=cluster 2024-09-18T01:17:13.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local instance_var_name=instance 2024-09-18T01:17:13.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ cluster=cluster1 2024-09-18T01:17:13.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ instance=0 2024-09-18T01:17:13.591 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:17:13.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ eval cluster=cluster1 2024-09-18T01:17:13.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++++ cluster=cluster1 2024-09-18T01:17:13.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ eval instance=0 2024-09-18T01:17:13.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++++ instance=0 2024-09-18T01:17:13.592 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2024-09-18T01:17:13.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2024-09-18T01:17:13.626 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2024-09-18T01:17:13.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ pid=29986 2024-09-18T01:17:13.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 29986 ']' 2024-09-18T01:17:13.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ kill 29986 2024-09-18T01:17:13.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 16 32 2024-09-18T01:17:13.635 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:17:14.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ps auxww 2024-09-18T01:17:14.637 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ awk -v pid=29986 '$2 == pid {print; exit 1}' 2024-09-18T01:17:14.719 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ break 2024-09-18T01:17:14.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ps auxww 2024-09-18T01:17:14.720 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ awk -v pid=29986 '$2 == pid {print; exit 1}' 2024-09-18T01:17:14.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:17:14.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local local_cluster=cluster1:0 2024-09-18T01:17:14.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local cluster=cluster1 2024-09-18T01:17:14.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local instance 2024-09-18T01:17:14.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:17:14.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local val=cluster1:0 2024-09-18T01:17:14.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local cluster_var_name=local_cluster 2024-09-18T01:17:14.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local instance_var_name=instance 2024-09-18T01:17:14.748 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ cluster=cluster1 2024-09-18T01:17:14.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ instance=0 2024-09-18T01:17:14.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:17:14.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ eval local_cluster=cluster1 2024-09-18T01:17:14.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local_cluster=cluster1 2024-09-18T01:17:14.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ eval instance=0 2024-09-18T01:17:14.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ instance=0 2024-09-18T01:17:14.749 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:17:14.772 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:17:14.773 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:17:14.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ daemon_asok_file cluster1:0 cluster2 2024-09-18T01:17:14.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local local_cluster=cluster1:0 2024-09-18T01:17:14.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local cluster=cluster2 2024-09-18T01:17:14.775 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local instance 2024-09-18T01:17:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:17:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local val=cluster1:0 2024-09-18T01:17:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local cluster_var_name=local_cluster 2024-09-18T01:17:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local instance_var_name=instance 2024-09-18T01:17:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ cluster=cluster1 2024-09-18T01:17:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ instance=0 2024-09-18T01:17:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:17:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ eval local_cluster=cluster1 2024-09-18T01:17:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local_cluster=cluster1 2024-09-18T01:17:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ eval instance=0 2024-09-18T01:17:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ instance=0 2024-09-18T01:17:14.776 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:17:14.805 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:17:14.806 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:17:14.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ daemon_pid_file cluster1:0 2024-09-18T01:17:14.808 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local cluster=cluster1:0 2024-09-18T01:17:14.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local instance 2024-09-18T01:17:14.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:17:14.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local val=cluster1:0 2024-09-18T01:17:14.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local cluster_var_name=cluster 2024-09-18T01:17:14.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local instance_var_name=instance 2024-09-18T01:17:14.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ cluster=cluster1 2024-09-18T01:17:14.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ instance=0 2024-09-18T01:17:14.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:17:14.809 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ eval cluster=cluster1 2024-09-18T01:17:14.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ cluster=cluster1 2024-09-18T01:17:14.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ eval instance=0 2024-09-18T01:17:14.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ instance=0 2024-09-18T01:17:14.810 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2024-09-18T01:17:14.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2024-09-18T01:17:14.835 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2024-09-18T01:17:14.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2024-09-18T01:17:14.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ stop_mirrors cluster2 2024-09-18T01:17:14.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2 2024-09-18T01:17:14.837 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local sig= 2024-09-18T01:17:14.838 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ seq 0 0 2024-09-18T01:17:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:17:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ stop_mirror cluster2:0 '' 2024-09-18T01:17:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local cluster=cluster2:0 2024-09-18T01:17:14.839 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local sig= 2024-09-18T01:17:14.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test -n '' 2024-09-18T01:17:14.840 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ local pid 2024-09-18T01:17:14.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ daemon_pid_file cluster2:0 2024-09-18T01:17:14.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local cluster=cluster2:0 2024-09-18T01:17:14.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local instance 2024-09-18T01:17:14.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2024-09-18T01:17:14.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local val=cluster2:0 2024-09-18T01:17:14.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local cluster_var_name=cluster 2024-09-18T01:17:14.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local instance_var_name=instance 2024-09-18T01:17:14.841 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ cluster=cluster2 2024-09-18T01:17:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ instance=0 2024-09-18T01:17:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ '[' 0 = cluster2:0 ']' 2024-09-18T01:17:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ eval cluster=cluster2 2024-09-18T01:17:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++++ cluster=cluster2 2024-09-18T01:17:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ eval instance=0 2024-09-18T01:17:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++++ instance=0 2024-09-18T01:17:14.842 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2024-09-18T01:17:14.869 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2024-09-18T01:17:14.870 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2024-09-18T01:17:14.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ pid=30015 2024-09-18T01:17:14.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -n 30015 ']' 2024-09-18T01:17:14.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ kill 30015 2024-09-18T01:17:14.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ for s in 1 2 4 8 16 32 2024-09-18T01:17:14.872 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ sleep 1 2024-09-18T01:17:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ps auxww 2024-09-18T01:17:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ awk -v pid=30015 '$2 == pid {print; exit 1}' 2024-09-18T01:17:15.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ break 2024-09-18T01:17:15.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ ps auxww 2024-09-18T01:17:15.902 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ awk -v pid=30015 '$2 == pid {print; exit 1}' 2024-09-18T01:17:15.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ daemon_asok_file cluster2:0 cluster1 2024-09-18T01:17:15.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local local_cluster=cluster2:0 2024-09-18T01:17:15.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local cluster=cluster1 2024-09-18T01:17:15.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local instance 2024-09-18T01:17:15.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2024-09-18T01:17:15.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local val=cluster2:0 2024-09-18T01:17:15.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local cluster_var_name=local_cluster 2024-09-18T01:17:15.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local instance_var_name=instance 2024-09-18T01:17:15.931 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ cluster=cluster2 2024-09-18T01:17:15.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ instance=0 2024-09-18T01:17:15.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ '[' 0 = cluster2:0 ']' 2024-09-18T01:17:15.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ eval local_cluster=cluster2 2024-09-18T01:17:15.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local_cluster=cluster2 2024-09-18T01:17:15.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ eval instance=0 2024-09-18T01:17:15.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ instance=0 2024-09-18T01:17:15.932 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2024-09-18T01:17:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2024-09-18T01:17:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2024-09-18T01:17:15.962 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ daemon_asok_file cluster2:0 cluster2 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local local_cluster=cluster2:0 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local cluster=cluster2 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local instance 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local val=cluster2:0 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local cluster_var_name=local_cluster 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local instance_var_name=instance 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ cluster=cluster2 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ instance=0 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ '[' 0 = cluster2:0 ']' 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ eval local_cluster=cluster2 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ local_cluster=cluster2 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ eval instance=0 2024-09-18T01:17:15.963 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ instance=0 2024-09-18T01:17:15.964 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2024-09-18T01:17:15.985 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2024-09-18T01:17:15.986 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2024-09-18T01:17:15.987 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ daemon_pid_file cluster2:0 2024-09-18T01:17:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local cluster=cluster2:0 2024-09-18T01:17:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local instance 2024-09-18T01:17:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ set_cluster_instance cluster2:0 cluster instance 2024-09-18T01:17:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local val=cluster2:0 2024-09-18T01:17:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local cluster_var_name=cluster 2024-09-18T01:17:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ local instance_var_name=instance 2024-09-18T01:17:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ cluster=cluster2 2024-09-18T01:17:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ instance=0 2024-09-18T01:17:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ '[' 0 = cluster2:0 ']' 2024-09-18T01:17:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ eval cluster=cluster2 2024-09-18T01:17:15.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ cluster=cluster2 2024-09-18T01:17:15.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ eval instance=0 2024-09-18T01:17:15.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ instance=0 2024-09-18T01:17:15.989 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2024-09-18T01:17:16.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2024-09-18T01:17:16.010 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2024-09-18T01:17:16.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' -z 1 ']' 2024-09-18T01:17:16.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ test '' = /tmp/tmp.Ir77rubWei 2024-09-18T01:17:16.011 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ rm -Rf /tmp/tmp.Ir77rubWei 2024-09-18T01:17:16.026 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ '[' 0 -eq 0 ']' 2024-09-18T01:17:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stdout:OK 2024-09-18T01:17:16.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ echo OK 2024-09-18T01:17:16.029 INFO:tasks.workunit.cluster1.client.mirror.smithi049.stderr:+ exit 0 2024-09-18T01:17:16.029 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-18T01:17:16.030 DEBUG:teuthology.orchestra.run.smithi049:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2024-09-18T01:17:16.147 INFO:tasks.workunit:Stopping ['rbd/compare_mirror_images.sh'] on cluster1.client.mirror... 2024-09-18T01:17:16.147 DEBUG:teuthology.orchestra.run.smithi049:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2024-09-18T01:17:16.536 DEBUG:teuthology.parallel:result is None 2024-09-18T01:17:16.537 DEBUG:teuthology.orchestra.run.smithi049:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2024-09-18T01:17:16.550 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2024-09-18T01:17:16.551 DEBUG:teuthology.orchestra.run.smithi049:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-18T01:17:16.597 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2024-09-18T01:17:16.597 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-09-18T01:17:16.609 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2024-09-18T01:17:16.609 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-18T01:17:16.977 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-18T01:17:16.978 INFO:teuthology.orchestra.run.smithi049.stderr:dumped all 2024-09-18T01:17:16.996 INFO:teuthology.orchestra.run.smithi049.stdout:{"pg_ready":true,"pg_map":{"version":4703,"stamp":"2024-09-18T01:17:16.127819+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":171,"num_write_kb":2550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":56,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3150764,"kb_used_data":17260,"kb_used_omap":4,"kb_used_meta":3133435,"kb_avail":279964756,"statfs":{"total":289910292480,"available":286683910144,"internally_reserved":0,"allocated":17674240,"data_stored":10315564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":3208637775},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":30,"apply_latency_ms":30,"commit_latency_ns":30000000,"apply_latency_ns":30000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-828926098,"num_objects":-256,"num_object_clones":-14,"num_object_copies":-512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-256,"num_whiteouts":0,"num_read":-1096920,"num_read_kb":-58317390,"num_write":-181486,"num_write_kb":-52085422,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-12,"num_bytes_recovered":-25219087,"num_keys_recovered":0,"num_objects_omap":-15,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.266462"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":3996,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562795+0000","last_change":"2024-09-17T23:46:43.368792+0000","last_active":"2024-09-18T01:17:14.562795+0000","last_peered":"2024-09-18T01:17:14.562795+0000","last_clean":"2024-09-18T01:17:14.562795+0000","last_became_active":"2024-09-17T23:46:41.368790+0000","last_became_peered":"2024-09-17T23:46:41.368790+0000","last_unstale":"2024-09-18T01:17:14.562795+0000","last_undegraded":"2024-09-18T01:17:14.562795+0000","last_fullsized":"2024-09-18T01:17:14.562795+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:34:56.033504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012361539999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":3996,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562676+0000","last_change":"2024-09-17T23:46:43.369747+0000","last_active":"2024-09-18T01:17:14.562676+0000","last_peered":"2024-09-18T01:17:14.562676+0000","last_clean":"2024-09-18T01:17:14.562676+0000","last_became_active":"2024-09-17T23:46:41.367907+0000","last_became_peered":"2024-09-17T23:46:41.367907+0000","last_unstale":"2024-09-18T01:17:14.562676+0000","last_undegraded":"2024-09-18T01:17:14.562676+0000","last_fullsized":"2024-09-18T01:17:14.562676+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:35:19.220254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002060517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":3998,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562628+0000","last_change":"2024-09-17T23:46:43.369633+0000","last_active":"2024-09-18T01:17:14.562628+0000","last_peered":"2024-09-18T01:17:14.562628+0000","last_clean":"2024-09-18T01:17:14.562628+0000","last_became_active":"2024-09-17T23:46:41.367679+0000","last_became_peered":"2024-09-17T23:46:41.367679+0000","last_unstale":"2024-09-18T01:17:14.562628+0000","last_undegraded":"2024-09-18T01:17:14.562628+0000","last_fullsized":"2024-09-18T01:17:14.562628+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:46:40.205238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020925570000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":3998,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562632+0000","last_change":"2024-09-17T23:46:43.368634+0000","last_active":"2024-09-18T01:17:14.562632+0000","last_peered":"2024-09-18T01:17:14.562632+0000","last_clean":"2024-09-18T01:17:14.562632+0000","last_became_active":"2024-09-17T23:46:41.368202+0000","last_became_peered":"2024-09-17T23:46:41.368202+0000","last_unstale":"2024-09-18T01:17:14.562632+0000","last_undegraded":"2024-09-18T01:17:14.562632+0000","last_fullsized":"2024-09-18T01:17:14.562632+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:30:19.644528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011726320000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":4000,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:13.453335+0000","last_change":"2024-09-17T23:46:43.373705+0000","last_active":"2024-09-18T01:17:13.453335+0000","last_peered":"2024-09-18T01:17:13.453335+0000","last_clean":"2024-09-18T01:17:13.453335+0000","last_became_active":"2024-09-17T23:46:41.366093+0000","last_became_peered":"2024-09-17T23:46:41.366093+0000","last_unstale":"2024-09-18T01:17:13.453335+0000","last_undegraded":"2024-09-18T01:17:13.453335+0000","last_fullsized":"2024-09-18T01:17:13.453335+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T00:00:49.511275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0021178640000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":3998,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:13.764217+0000","last_change":"2024-09-17T23:46:43.865198+0000","last_active":"2024-09-18T01:17:13.764217+0000","last_peered":"2024-09-18T01:17:13.764217+0000","last_clean":"2024-09-18T01:17:13.764217+0000","last_became_active":"2024-09-17T23:46:41.367488+0000","last_became_peered":"2024-09-17T23:46:41.367488+0000","last_unstale":"2024-09-18T01:17:13.764217+0000","last_undegraded":"2024-09-18T01:17:13.764217+0000","last_fullsized":"2024-09-18T01:17:13.764217+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:39:09.298304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011158380000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":4000,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:13.764932+0000","last_change":"2024-09-17T23:46:43.865240+0000","last_active":"2024-09-18T01:17:13.764932+0000","last_peered":"2024-09-18T01:17:13.764932+0000","last_clean":"2024-09-18T01:17:13.764932+0000","last_became_active":"2024-09-17T23:46:41.365093+0000","last_became_peered":"2024-09-17T23:46:41.365093+0000","last_unstale":"2024-09-18T01:17:13.764932+0000","last_undegraded":"2024-09-18T01:17:13.764932+0000","last_fullsized":"2024-09-18T01:17:13.764932+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T23:46:54.488828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011454340000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":3999,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.575362+0000","last_change":"2024-09-17T23:46:43.368031+0000","last_active":"2024-09-18T01:17:14.575362+0000","last_peered":"2024-09-18T01:17:14.575362+0000","last_clean":"2024-09-18T01:17:14.575362+0000","last_became_active":"2024-09-17T23:46:41.364533+0000","last_became_peered":"2024-09-17T23:46:41.364533+0000","last_unstale":"2024-09-18T01:17:14.575362+0000","last_undegraded":"2024-09-18T01:17:14.575362+0000","last_fullsized":"2024-09-18T01:17:14.575362+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:27:19.243264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077234799999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"464'125","reported_seq":4162,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.571013+0000","last_change":"2024-09-17T23:46:37.589635+0000","last_active":"2024-09-18T01:17:14.571013+0000","last_peered":"2024-09-18T01:17:14.571013+0000","last_clean":"2024-09-18T01:17:14.571013+0000","last_became_active":"2024-09-17T23:46:37.587086+0000","last_became_peered":"2024-09-17T23:46:37.587086+0000","last_unstale":"2024-09-18T01:17:14.571013+0000","last_undegraded":"2024-09-18T01:17:14.571013+0000","last_fullsized":"2024-09-18T01:17:14.571013+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:36.327066+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:36.327066+0000","last_clean_scrub_stamp":"2024-09-17T23:46:36.327066+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:04:13.926025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125,"ondisk_log_size":125,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706760,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1024840,"kb_used_data":5364,"kb_used_omap":1,"kb_used_meta":1019454,"kb_avail":93347000,"statfs":{"total":96636764160,"available":95587328000,"internally_reserved":0,"allocated":5492736,"data_stored":3044930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1043921349},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706756,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":997648,"kb_used_data":5948,"kb_used_omap":1,"kb_used_meta":991678,"kb_avail":93374192,"statfs":{"total":96636764160,"available":95615172608,"internally_reserved":0,"allocated":6090752,"data_stored":3635317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1015478725},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706760,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1128276,"kb_used_data":5948,"kb_used_omap":1,"kb_used_meta":1122302,"kb_avail":93243564,"statfs":{"total":96636764160,"available":95481409536,"internally_reserved":0,"allocated":6090752,"data_stored":3635317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1149237701},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:17:16.997 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-18T01:17:17.290 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-18T01:17:17.290 INFO:teuthology.orchestra.run.smithi049.stderr:dumped all 2024-09-18T01:17:17.308 INFO:teuthology.orchestra.run.smithi049.stdout:{"pg_ready":true,"pg_map":{"version":4703,"stamp":"2024-09-18T01:17:16.127819+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":171,"num_write_kb":2550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":56,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3150764,"kb_used_data":17260,"kb_used_omap":4,"kb_used_meta":3133435,"kb_avail":279964756,"statfs":{"total":289910292480,"available":286683910144,"internally_reserved":0,"allocated":17674240,"data_stored":10315564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":3208637775},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":30,"apply_latency_ms":30,"commit_latency_ns":30000000,"apply_latency_ns":30000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-828926098,"num_objects":-256,"num_object_clones":-14,"num_object_copies":-512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-256,"num_whiteouts":0,"num_read":-1096920,"num_read_kb":-58317390,"num_write":-181486,"num_write_kb":-52085422,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-12,"num_bytes_recovered":-25219087,"num_keys_recovered":0,"num_objects_omap":-15,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.266462"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":3996,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562795+0000","last_change":"2024-09-17T23:46:43.368792+0000","last_active":"2024-09-18T01:17:14.562795+0000","last_peered":"2024-09-18T01:17:14.562795+0000","last_clean":"2024-09-18T01:17:14.562795+0000","last_became_active":"2024-09-17T23:46:41.368790+0000","last_became_peered":"2024-09-17T23:46:41.368790+0000","last_unstale":"2024-09-18T01:17:14.562795+0000","last_undegraded":"2024-09-18T01:17:14.562795+0000","last_fullsized":"2024-09-18T01:17:14.562795+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:34:56.033504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012361539999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":3996,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562676+0000","last_change":"2024-09-17T23:46:43.369747+0000","last_active":"2024-09-18T01:17:14.562676+0000","last_peered":"2024-09-18T01:17:14.562676+0000","last_clean":"2024-09-18T01:17:14.562676+0000","last_became_active":"2024-09-17T23:46:41.367907+0000","last_became_peered":"2024-09-17T23:46:41.367907+0000","last_unstale":"2024-09-18T01:17:14.562676+0000","last_undegraded":"2024-09-18T01:17:14.562676+0000","last_fullsized":"2024-09-18T01:17:14.562676+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:35:19.220254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002060517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":3998,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562628+0000","last_change":"2024-09-17T23:46:43.369633+0000","last_active":"2024-09-18T01:17:14.562628+0000","last_peered":"2024-09-18T01:17:14.562628+0000","last_clean":"2024-09-18T01:17:14.562628+0000","last_became_active":"2024-09-17T23:46:41.367679+0000","last_became_peered":"2024-09-17T23:46:41.367679+0000","last_unstale":"2024-09-18T01:17:14.562628+0000","last_undegraded":"2024-09-18T01:17:14.562628+0000","last_fullsized":"2024-09-18T01:17:14.562628+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:46:40.205238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020925570000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":3998,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562632+0000","last_change":"2024-09-17T23:46:43.368634+0000","last_active":"2024-09-18T01:17:14.562632+0000","last_peered":"2024-09-18T01:17:14.562632+0000","last_clean":"2024-09-18T01:17:14.562632+0000","last_became_active":"2024-09-17T23:46:41.368202+0000","last_became_peered":"2024-09-17T23:46:41.368202+0000","last_unstale":"2024-09-18T01:17:14.562632+0000","last_undegraded":"2024-09-18T01:17:14.562632+0000","last_fullsized":"2024-09-18T01:17:14.562632+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:30:19.644528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011726320000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":4000,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:13.453335+0000","last_change":"2024-09-17T23:46:43.373705+0000","last_active":"2024-09-18T01:17:13.453335+0000","last_peered":"2024-09-18T01:17:13.453335+0000","last_clean":"2024-09-18T01:17:13.453335+0000","last_became_active":"2024-09-17T23:46:41.366093+0000","last_became_peered":"2024-09-17T23:46:41.366093+0000","last_unstale":"2024-09-18T01:17:13.453335+0000","last_undegraded":"2024-09-18T01:17:13.453335+0000","last_fullsized":"2024-09-18T01:17:13.453335+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T00:00:49.511275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0021178640000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":3998,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:13.764217+0000","last_change":"2024-09-17T23:46:43.865198+0000","last_active":"2024-09-18T01:17:13.764217+0000","last_peered":"2024-09-18T01:17:13.764217+0000","last_clean":"2024-09-18T01:17:13.764217+0000","last_became_active":"2024-09-17T23:46:41.367488+0000","last_became_peered":"2024-09-17T23:46:41.367488+0000","last_unstale":"2024-09-18T01:17:13.764217+0000","last_undegraded":"2024-09-18T01:17:13.764217+0000","last_fullsized":"2024-09-18T01:17:13.764217+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:39:09.298304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011158380000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":4000,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:13.764932+0000","last_change":"2024-09-17T23:46:43.865240+0000","last_active":"2024-09-18T01:17:13.764932+0000","last_peered":"2024-09-18T01:17:13.764932+0000","last_clean":"2024-09-18T01:17:13.764932+0000","last_became_active":"2024-09-17T23:46:41.365093+0000","last_became_peered":"2024-09-17T23:46:41.365093+0000","last_unstale":"2024-09-18T01:17:13.764932+0000","last_undegraded":"2024-09-18T01:17:13.764932+0000","last_fullsized":"2024-09-18T01:17:13.764932+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T23:46:54.488828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011454340000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":3999,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.575362+0000","last_change":"2024-09-17T23:46:43.368031+0000","last_active":"2024-09-18T01:17:14.575362+0000","last_peered":"2024-09-18T01:17:14.575362+0000","last_clean":"2024-09-18T01:17:14.575362+0000","last_became_active":"2024-09-17T23:46:41.364533+0000","last_became_peered":"2024-09-17T23:46:41.364533+0000","last_unstale":"2024-09-18T01:17:14.575362+0000","last_undegraded":"2024-09-18T01:17:14.575362+0000","last_fullsized":"2024-09-18T01:17:14.575362+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:27:19.243264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077234799999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"464'125","reported_seq":4162,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.571013+0000","last_change":"2024-09-17T23:46:37.589635+0000","last_active":"2024-09-18T01:17:14.571013+0000","last_peered":"2024-09-18T01:17:14.571013+0000","last_clean":"2024-09-18T01:17:14.571013+0000","last_became_active":"2024-09-17T23:46:37.587086+0000","last_became_peered":"2024-09-17T23:46:37.587086+0000","last_unstale":"2024-09-18T01:17:14.571013+0000","last_undegraded":"2024-09-18T01:17:14.571013+0000","last_fullsized":"2024-09-18T01:17:14.571013+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:36.327066+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:36.327066+0000","last_clean_scrub_stamp":"2024-09-17T23:46:36.327066+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:04:13.926025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125,"ondisk_log_size":125,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706760,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1024840,"kb_used_data":5364,"kb_used_omap":1,"kb_used_meta":1019454,"kb_avail":93347000,"statfs":{"total":96636764160,"available":95587328000,"internally_reserved":0,"allocated":5492736,"data_stored":3044930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1043921349},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706756,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":997648,"kb_used_data":5948,"kb_used_omap":1,"kb_used_meta":991678,"kb_avail":93374192,"statfs":{"total":96636764160,"available":95615172608,"internally_reserved":0,"allocated":6090752,"data_stored":3635317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1015478725},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706760,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1128276,"kb_used_data":5948,"kb_used_omap":1,"kb_used_meta":1122302,"kb_avail":93243564,"statfs":{"total":96636764160,"available":95481409536,"internally_reserved":0,"allocated":6090752,"data_stored":3635317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1149237701},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:17:17.309 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2024-09-18T01:17:17.309 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-18T01:17:17.609 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-18T01:17:17.609 INFO:teuthology.orchestra.run.smithi049.stderr:dumped all 2024-09-18T01:17:17.627 INFO:teuthology.orchestra.run.smithi049.stdout:{"pg_ready":true,"pg_map":{"version":4703,"stamp":"2024-09-18T01:17:16.127819+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":171,"num_write_kb":2550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":56,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3150764,"kb_used_data":17260,"kb_used_omap":4,"kb_used_meta":3133435,"kb_avail":279964756,"statfs":{"total":289910292480,"available":286683910144,"internally_reserved":0,"allocated":17674240,"data_stored":10315564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":3208637775},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":30,"apply_latency_ms":30,"commit_latency_ns":30000000,"apply_latency_ns":30000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-828926098,"num_objects":-256,"num_object_clones":-14,"num_object_copies":-512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-256,"num_whiteouts":0,"num_read":-1096920,"num_read_kb":-58317390,"num_write":-181486,"num_write_kb":-52085422,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-12,"num_bytes_recovered":-25219087,"num_keys_recovered":0,"num_objects_omap":-15,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.266462"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":3996,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562795+0000","last_change":"2024-09-17T23:46:43.368792+0000","last_active":"2024-09-18T01:17:14.562795+0000","last_peered":"2024-09-18T01:17:14.562795+0000","last_clean":"2024-09-18T01:17:14.562795+0000","last_became_active":"2024-09-17T23:46:41.368790+0000","last_became_peered":"2024-09-17T23:46:41.368790+0000","last_unstale":"2024-09-18T01:17:14.562795+0000","last_undegraded":"2024-09-18T01:17:14.562795+0000","last_fullsized":"2024-09-18T01:17:14.562795+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:34:56.033504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012361539999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":3996,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562676+0000","last_change":"2024-09-17T23:46:43.369747+0000","last_active":"2024-09-18T01:17:14.562676+0000","last_peered":"2024-09-18T01:17:14.562676+0000","last_clean":"2024-09-18T01:17:14.562676+0000","last_became_active":"2024-09-17T23:46:41.367907+0000","last_became_peered":"2024-09-17T23:46:41.367907+0000","last_unstale":"2024-09-18T01:17:14.562676+0000","last_undegraded":"2024-09-18T01:17:14.562676+0000","last_fullsized":"2024-09-18T01:17:14.562676+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:35:19.220254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002060517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":3998,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562628+0000","last_change":"2024-09-17T23:46:43.369633+0000","last_active":"2024-09-18T01:17:14.562628+0000","last_peered":"2024-09-18T01:17:14.562628+0000","last_clean":"2024-09-18T01:17:14.562628+0000","last_became_active":"2024-09-17T23:46:41.367679+0000","last_became_peered":"2024-09-17T23:46:41.367679+0000","last_unstale":"2024-09-18T01:17:14.562628+0000","last_undegraded":"2024-09-18T01:17:14.562628+0000","last_fullsized":"2024-09-18T01:17:14.562628+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:46:40.205238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020925570000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":3998,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562632+0000","last_change":"2024-09-17T23:46:43.368634+0000","last_active":"2024-09-18T01:17:14.562632+0000","last_peered":"2024-09-18T01:17:14.562632+0000","last_clean":"2024-09-18T01:17:14.562632+0000","last_became_active":"2024-09-17T23:46:41.368202+0000","last_became_peered":"2024-09-17T23:46:41.368202+0000","last_unstale":"2024-09-18T01:17:14.562632+0000","last_undegraded":"2024-09-18T01:17:14.562632+0000","last_fullsized":"2024-09-18T01:17:14.562632+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:30:19.644528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011726320000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":4000,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:13.453335+0000","last_change":"2024-09-17T23:46:43.373705+0000","last_active":"2024-09-18T01:17:13.453335+0000","last_peered":"2024-09-18T01:17:13.453335+0000","last_clean":"2024-09-18T01:17:13.453335+0000","last_became_active":"2024-09-17T23:46:41.366093+0000","last_became_peered":"2024-09-17T23:46:41.366093+0000","last_unstale":"2024-09-18T01:17:13.453335+0000","last_undegraded":"2024-09-18T01:17:13.453335+0000","last_fullsized":"2024-09-18T01:17:13.453335+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T00:00:49.511275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0021178640000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":3998,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:13.764217+0000","last_change":"2024-09-17T23:46:43.865198+0000","last_active":"2024-09-18T01:17:13.764217+0000","last_peered":"2024-09-18T01:17:13.764217+0000","last_clean":"2024-09-18T01:17:13.764217+0000","last_became_active":"2024-09-17T23:46:41.367488+0000","last_became_peered":"2024-09-17T23:46:41.367488+0000","last_unstale":"2024-09-18T01:17:13.764217+0000","last_undegraded":"2024-09-18T01:17:13.764217+0000","last_fullsized":"2024-09-18T01:17:13.764217+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:39:09.298304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011158380000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":4000,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:13.764932+0000","last_change":"2024-09-17T23:46:43.865240+0000","last_active":"2024-09-18T01:17:13.764932+0000","last_peered":"2024-09-18T01:17:13.764932+0000","last_clean":"2024-09-18T01:17:13.764932+0000","last_became_active":"2024-09-17T23:46:41.365093+0000","last_became_peered":"2024-09-17T23:46:41.365093+0000","last_unstale":"2024-09-18T01:17:13.764932+0000","last_undegraded":"2024-09-18T01:17:13.764932+0000","last_fullsized":"2024-09-18T01:17:13.764932+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T23:46:54.488828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011454340000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":3999,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.575362+0000","last_change":"2024-09-17T23:46:43.368031+0000","last_active":"2024-09-18T01:17:14.575362+0000","last_peered":"2024-09-18T01:17:14.575362+0000","last_clean":"2024-09-18T01:17:14.575362+0000","last_became_active":"2024-09-17T23:46:41.364533+0000","last_became_peered":"2024-09-17T23:46:41.364533+0000","last_unstale":"2024-09-18T01:17:14.575362+0000","last_undegraded":"2024-09-18T01:17:14.575362+0000","last_fullsized":"2024-09-18T01:17:14.575362+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:27:19.243264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077234799999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"464'125","reported_seq":4162,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.571013+0000","last_change":"2024-09-17T23:46:37.589635+0000","last_active":"2024-09-18T01:17:14.571013+0000","last_peered":"2024-09-18T01:17:14.571013+0000","last_clean":"2024-09-18T01:17:14.571013+0000","last_became_active":"2024-09-17T23:46:37.587086+0000","last_became_peered":"2024-09-17T23:46:37.587086+0000","last_unstale":"2024-09-18T01:17:14.571013+0000","last_undegraded":"2024-09-18T01:17:14.571013+0000","last_fullsized":"2024-09-18T01:17:14.571013+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:36.327066+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:36.327066+0000","last_clean_scrub_stamp":"2024-09-17T23:46:36.327066+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:04:13.926025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125,"ondisk_log_size":125,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706760,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1024840,"kb_used_data":5364,"kb_used_omap":1,"kb_used_meta":1019454,"kb_avail":93347000,"statfs":{"total":96636764160,"available":95587328000,"internally_reserved":0,"allocated":5492736,"data_stored":3044930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1043921349},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706756,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":997648,"kb_used_data":5948,"kb_used_omap":1,"kb_used_meta":991678,"kb_avail":93374192,"statfs":{"total":96636764160,"available":95615172608,"internally_reserved":0,"allocated":6090752,"data_stored":3635317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1015478725},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706760,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1128276,"kb_used_data":5948,"kb_used_omap":1,"kb_used_meta":1122302,"kb_avail":93243564,"statfs":{"total":96636764160,"available":95481409536,"internally_reserved":0,"allocated":6090752,"data_stored":3635317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1149237701},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:17:17.627 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-18T01:17:17.917 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-18T01:17:17.917 INFO:teuthology.orchestra.run.smithi049.stdout:{"epoch":1985,"fsid":"024a7a4f-3e37-4953-ac7d-ad33b26dedaa","created":"2024-09-17T23:46:24.453844+0000","modified":"2024-09-18T01:17:13.437081+0000","last_up_change":"2024-09-17T23:46:34.324287+0000","last_in_change":"2024-09-17T23:46:25.380066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T23:46:36.186072+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-17T23:46:39.870458+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6fde4361-75a8-478d-b104-6083fe26abe3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6800","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6801","nonce":687973982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6802","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6803","nonce":687973982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6806","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6807","nonce":687973982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6804","nonce":687973982},{"type":"v1","addr":"172.21.15.49:6805","nonce":687973982}]},"public_addr":"172.21.15.49:6801/687973982","cluster_addr":"172.21.15.49:6803/687973982","heartbeat_back_addr":"172.21.15.49:6807/687973982","heartbeat_front_addr":"172.21.15.49:6805/687973982","state":["exists","up"]},{"osd":1,"uuid":"b54753b9-dd30-4601-93f0-bf45a643a1d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6816","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6817","nonce":965597954}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6818","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6819","nonce":965597954}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6822","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6823","nonce":965597954}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6820","nonce":965597954},{"type":"v1","addr":"172.21.15.49:6821","nonce":965597954}]},"public_addr":"172.21.15.49:6817/965597954","cluster_addr":"172.21.15.49:6819/965597954","heartbeat_back_addr":"172.21.15.49:6823/965597954","heartbeat_front_addr":"172.21.15.49:6821/965597954","state":["exists","up"]},{"osd":2,"uuid":"5fdd2e96-616c-48e2-8473-0f00c1016b6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":38,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6808","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6809","nonce":130731046}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6810","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6811","nonce":130731046}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6814","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6815","nonce":130731046}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.49:6812","nonce":130731046},{"type":"v1","addr":"172.21.15.49:6813","nonce":130731046}]},"public_addr":"172.21.15.49:6809/130731046","cluster_addr":"172.21.15.49:6811/130731046","heartbeat_back_addr":"172.21.15.49:6815/130731046","heartbeat_front_addr":"172.21.15.49:6813/130731046","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:46:31.554586+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:46:32.203336+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:46:31.945325+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":3301,"length":1},{"begin":3308,"length":2},{"begin":3314,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T01:17:18.938 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2024-09-18T01:17:18.938 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-09-18T01:17:19.106 INFO:teuthology.orchestra.run.smithi049.stdout:{ 2024-09-18T01:17:19.106 INFO:teuthology.orchestra.run.smithi049.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-18T01:17:19.106 INFO:teuthology.orchestra.run.smithi049.stdout:} 2024-09-18T01:17:19.123 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2024-09-18T01:17:19.411 INFO:teuthology.orchestra.run.smithi049.stderr:instructed osd(s) 0 to deep-scrub 2024-09-18T01:17:19.429 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2024-09-18T01:17:19.429 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-09-18T01:17:19.580 INFO:teuthology.orchestra.run.smithi049.stdout:{ 2024-09-18T01:17:19.580 INFO:teuthology.orchestra.run.smithi049.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-18T01:17:19.580 INFO:teuthology.orchestra.run.smithi049.stdout:} 2024-09-18T01:17:19.597 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2024-09-18T01:17:19.888 INFO:teuthology.orchestra.run.smithi049.stderr:instructed osd(s) 1 to deep-scrub 2024-09-18T01:17:19.905 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2024-09-18T01:17:19.906 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-09-18T01:17:20.045 INFO:teuthology.orchestra.run.smithi049.stdout:{ 2024-09-18T01:17:20.045 INFO:teuthology.orchestra.run.smithi049.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-18T01:17:20.046 INFO:teuthology.orchestra.run.smithi049.stdout:} 2024-09-18T01:17:20.060 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2024-09-18T01:17:20.346 INFO:teuthology.orchestra.run.smithi049.stderr:instructed osd(s) 2 to deep-scrub 2024-09-18T01:17:20.365 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-18T01:17:20.647 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-18T01:17:20.647 INFO:teuthology.orchestra.run.smithi049.stderr:dumped all 2024-09-18T01:17:20.664 INFO:teuthology.orchestra.run.smithi049.stdout:{"pg_ready":true,"pg_map":{"version":4705,"stamp":"2024-09-18T01:17:20.128804+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":171,"num_write_kb":2550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":56,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3150764,"kb_used_data":17260,"kb_used_omap":4,"kb_used_meta":3133435,"kb_avail":279964756,"statfs":{"total":289910292480,"available":286683910144,"internally_reserved":0,"allocated":17674240,"data_stored":10315564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":3208637775},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":30,"apply_latency_ms":30,"commit_latency_ns":30000000,"apply_latency_ns":30000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-19,"num_objects":-5,"num_object_clones":0,"num_object_copies":-10,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5,"num_whiteouts":0,"num_read":-2874,"num_read_kb":-2868,"num_write":-187,"num_write_kb":-185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.272521"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":3996,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562795+0000","last_change":"2024-09-17T23:46:43.368792+0000","last_active":"2024-09-18T01:17:14.562795+0000","last_peered":"2024-09-18T01:17:14.562795+0000","last_clean":"2024-09-18T01:17:14.562795+0000","last_became_active":"2024-09-17T23:46:41.368790+0000","last_became_peered":"2024-09-17T23:46:41.368790+0000","last_unstale":"2024-09-18T01:17:14.562795+0000","last_undegraded":"2024-09-18T01:17:14.562795+0000","last_fullsized":"2024-09-18T01:17:14.562795+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:34:56.033504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012361539999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":3996,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562676+0000","last_change":"2024-09-17T23:46:43.369747+0000","last_active":"2024-09-18T01:17:14.562676+0000","last_peered":"2024-09-18T01:17:14.562676+0000","last_clean":"2024-09-18T01:17:14.562676+0000","last_became_active":"2024-09-17T23:46:41.367907+0000","last_became_peered":"2024-09-17T23:46:41.367907+0000","last_unstale":"2024-09-18T01:17:14.562676+0000","last_undegraded":"2024-09-18T01:17:14.562676+0000","last_fullsized":"2024-09-18T01:17:14.562676+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:35:19.220254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002060517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":3998,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562628+0000","last_change":"2024-09-17T23:46:43.369633+0000","last_active":"2024-09-18T01:17:14.562628+0000","last_peered":"2024-09-18T01:17:14.562628+0000","last_clean":"2024-09-18T01:17:14.562628+0000","last_became_active":"2024-09-17T23:46:41.367679+0000","last_became_peered":"2024-09-17T23:46:41.367679+0000","last_unstale":"2024-09-18T01:17:14.562628+0000","last_undegraded":"2024-09-18T01:17:14.562628+0000","last_fullsized":"2024-09-18T01:17:14.562628+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:46:40.205238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020925570000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":3998,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.562632+0000","last_change":"2024-09-17T23:46:43.368634+0000","last_active":"2024-09-18T01:17:14.562632+0000","last_peered":"2024-09-18T01:17:14.562632+0000","last_clean":"2024-09-18T01:17:14.562632+0000","last_became_active":"2024-09-17T23:46:41.368202+0000","last_became_peered":"2024-09-17T23:46:41.368202+0000","last_unstale":"2024-09-18T01:17:14.562632+0000","last_undegraded":"2024-09-18T01:17:14.562632+0000","last_fullsized":"2024-09-18T01:17:14.562632+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:30:19.644528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011726320000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":4000,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:13.453335+0000","last_change":"2024-09-17T23:46:43.373705+0000","last_active":"2024-09-18T01:17:13.453335+0000","last_peered":"2024-09-18T01:17:13.453335+0000","last_clean":"2024-09-18T01:17:13.453335+0000","last_became_active":"2024-09-17T23:46:41.366093+0000","last_became_peered":"2024-09-17T23:46:41.366093+0000","last_unstale":"2024-09-18T01:17:13.453335+0000","last_undegraded":"2024-09-18T01:17:13.453335+0000","last_fullsized":"2024-09-18T01:17:13.453335+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T00:00:49.511275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0021178640000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":3998,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:13.764217+0000","last_change":"2024-09-17T23:46:43.865198+0000","last_active":"2024-09-18T01:17:13.764217+0000","last_peered":"2024-09-18T01:17:13.764217+0000","last_clean":"2024-09-18T01:17:13.764217+0000","last_became_active":"2024-09-17T23:46:41.367488+0000","last_became_peered":"2024-09-17T23:46:41.367488+0000","last_unstale":"2024-09-18T01:17:13.764217+0000","last_undegraded":"2024-09-18T01:17:13.764217+0000","last_fullsized":"2024-09-18T01:17:13.764217+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:39:09.298304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011158380000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":4000,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:13.764932+0000","last_change":"2024-09-17T23:46:43.865240+0000","last_active":"2024-09-18T01:17:13.764932+0000","last_peered":"2024-09-18T01:17:13.764932+0000","last_clean":"2024-09-18T01:17:13.764932+0000","last_became_active":"2024-09-17T23:46:41.365093+0000","last_became_peered":"2024-09-17T23:46:41.365093+0000","last_unstale":"2024-09-18T01:17:13.764932+0000","last_undegraded":"2024-09-18T01:17:13.764932+0000","last_fullsized":"2024-09-18T01:17:13.764932+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T23:46:54.488828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011454340000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":3999,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.575362+0000","last_change":"2024-09-17T23:46:43.368031+0000","last_active":"2024-09-18T01:17:14.575362+0000","last_peered":"2024-09-18T01:17:14.575362+0000","last_clean":"2024-09-18T01:17:14.575362+0000","last_became_active":"2024-09-17T23:46:41.364533+0000","last_became_peered":"2024-09-17T23:46:41.364533+0000","last_unstale":"2024-09-18T01:17:14.575362+0000","last_undegraded":"2024-09-18T01:17:14.575362+0000","last_fullsized":"2024-09-18T01:17:14.575362+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:40.343119+0000","last_clean_scrub_stamp":"2024-09-17T23:46:40.343119+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:27:19.243264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077234799999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"464'125","reported_seq":4162,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:14.571013+0000","last_change":"2024-09-17T23:46:37.589635+0000","last_active":"2024-09-18T01:17:14.571013+0000","last_peered":"2024-09-18T01:17:14.571013+0000","last_clean":"2024-09-18T01:17:14.571013+0000","last_became_active":"2024-09-17T23:46:37.587086+0000","last_became_peered":"2024-09-17T23:46:37.587086+0000","last_unstale":"2024-09-18T01:17:14.571013+0000","last_undegraded":"2024-09-18T01:17:14.571013+0000","last_fullsized":"2024-09-18T01:17:14.571013+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:36.327066+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:36.327066+0000","last_clean_scrub_stamp":"2024-09-17T23:46:36.327066+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:04:13.926025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125,"ondisk_log_size":125,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706760,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1024840,"kb_used_data":5364,"kb_used_omap":1,"kb_used_meta":1019454,"kb_avail":93347000,"statfs":{"total":96636764160,"available":95587328000,"internally_reserved":0,"allocated":5492736,"data_stored":3044930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1043921349},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706756,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":997648,"kb_used_data":5948,"kb_used_omap":1,"kb_used_meta":991678,"kb_avail":93374192,"statfs":{"total":96636764160,"available":95615172608,"internally_reserved":0,"allocated":6090752,"data_stored":3635317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1015478725},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706760,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1128276,"kb_used_data":5948,"kb_used_omap":1,"kb_used_meta":1122302,"kb_avail":93243564,"statfs":{"total":96636764160,"available":95481409536,"internally_reserved":0,"allocated":6090752,"data_stored":3635317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1149237701},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:17:20.668 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2024-09-17T23:46:40.343119+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=40, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=17, tm_sec=17, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:17:20.668 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2024-09-17T23:46:40.343119+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=40, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=17, tm_sec=17, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:17:20.668 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2024-09-17T23:46:40.343119+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=40, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=17, tm_sec=17, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:17:20.668 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2024-09-17T23:46:40.343119+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=40, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=17, tm_sec=17, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:17:20.668 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2024-09-17T23:46:40.343119+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=40, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=17, tm_sec=17, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:17:20.668 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2024-09-17T23:46:40.343119+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=40, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=17, tm_sec=17, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:17:20.668 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2024-09-17T23:46:40.343119+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=40, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=17, tm_sec=17, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:17:20.669 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2024-09-17T23:46:40.343119+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=40, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=17, tm_sec=17, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:17:20.669 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-09-17T23:46:36.327066+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=36, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=17, tm_sec=17, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:17:20.669 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-09-18T01:17:40.669 DEBUG:teuthology.orchestra.run.smithi049:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-18T01:17:40.988 INFO:teuthology.orchestra.run.smithi049.stdout: 2024-09-18T01:17:40.989 INFO:teuthology.orchestra.run.smithi049.stderr:dumped all 2024-09-18T01:17:41.007 INFO:teuthology.orchestra.run.smithi049.stdout:{"pg_ready":true,"pg_map":{"version":4715,"stamp":"2024-09-18T01:17:40.133641+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":171,"num_write_kb":2550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132,"ondisk_log_size":132,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":3150748,"kb_used_data":17260,"kb_used_omap":4,"kb_used_meta":3133435,"kb_avail":279964772,"statfs":{"total":289910292480,"available":286683926528,"internally_reserved":0,"allocated":17674240,"data_stored":10315564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":3208637775},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002878"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":4005,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:24.625767+0000","last_change":"2024-09-18T01:17:24.625898+0000","last_active":"2024-09-18T01:17:24.625767+0000","last_peered":"2024-09-18T01:17:24.625767+0000","last_clean":"2024-09-18T01:17:24.625767+0000","last_became_active":"2024-09-17T23:46:41.368790+0000","last_became_peered":"2024-09-17T23:46:41.368790+0000","last_unstale":"2024-09-18T01:17:24.625767+0000","last_undegraded":"2024-09-18T01:17:24.625767+0000","last_fullsized":"2024-09-18T01:17:24.625767+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:17:24.625533+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:17:24.625533+0000","last_clean_scrub_stamp":"2024-09-18T01:17:24.625533+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:38:57.059668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012361539999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":4005,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:23.614607+0000","last_change":"2024-09-18T01:17:23.614663+0000","last_active":"2024-09-18T01:17:23.614607+0000","last_peered":"2024-09-18T01:17:23.614607+0000","last_clean":"2024-09-18T01:17:23.614607+0000","last_became_active":"2024-09-17T23:46:41.367907+0000","last_became_peered":"2024-09-17T23:46:41.367907+0000","last_unstale":"2024-09-18T01:17:23.614607+0000","last_undegraded":"2024-09-18T01:17:23.614607+0000","last_fullsized":"2024-09-18T01:17:23.614607+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:17:23.614412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:17:23.614412+0000","last_clean_scrub_stamp":"2024-09-18T01:17:23.614412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:29:39.151259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002060517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":4007,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:22.583315+0000","last_change":"2024-09-18T01:17:22.583369+0000","last_active":"2024-09-18T01:17:22.583315+0000","last_peered":"2024-09-18T01:17:22.583315+0000","last_clean":"2024-09-18T01:17:22.583315+0000","last_became_active":"2024-09-17T23:46:41.367679+0000","last_became_peered":"2024-09-17T23:46:41.367679+0000","last_unstale":"2024-09-18T01:17:22.583315+0000","last_undegraded":"2024-09-18T01:17:22.583315+0000","last_fullsized":"2024-09-18T01:17:22.583315+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:17:22.583103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:17:22.583103+0000","last_clean_scrub_stamp":"2024-09-18T01:17:22.583103+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:53:49.186670+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.0020925570000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":4007,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:25.623141+0000","last_change":"2024-09-18T01:17:25.623198+0000","last_active":"2024-09-18T01:17:25.623141+0000","last_peered":"2024-09-18T01:17:25.623141+0000","last_clean":"2024-09-18T01:17:25.623141+0000","last_became_active":"2024-09-17T23:46:41.368202+0000","last_became_peered":"2024-09-17T23:46:41.368202+0000","last_unstale":"2024-09-18T01:17:25.623141+0000","last_undegraded":"2024-09-18T01:17:25.623141+0000","last_fullsized":"2024-09-18T01:17:25.623141+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:17:25.622936+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:17:25.622936+0000","last_clean_scrub_stamp":"2024-09-18T01:17:25.622936+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:26:14.917580+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.0011726320000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"1985'3","reported_seq":4010,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:19.434272+0000","last_change":"2024-09-18T01:17:19.434320+0000","last_active":"2024-09-18T01:17:19.434272+0000","last_peered":"2024-09-18T01:17:19.434272+0000","last_clean":"2024-09-18T01:17:19.434272+0000","last_became_active":"2024-09-17T23:46:41.366093+0000","last_became_peered":"2024-09-17T23:46:41.366093+0000","last_unstale":"2024-09-18T01:17:19.434272+0000","last_undegraded":"2024-09-18T01:17:19.434272+0000","last_fullsized":"2024-09-18T01:17:19.434272+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"1985'3","last_scrub_stamp":"2024-09-18T01:17:19.434169+0000","last_deep_scrub":"1985'3","last_deep_scrub_stamp":"2024-09-18T01:17:19.434169+0000","last_clean_scrub_stamp":"2024-09-18T01:17:19.434169+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:01:07.147929+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.0021178640000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":4007,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:21.377936+0000","last_change":"2024-09-18T01:17:21.377993+0000","last_active":"2024-09-18T01:17:21.377936+0000","last_peered":"2024-09-18T01:17:21.377936+0000","last_clean":"2024-09-18T01:17:21.377936+0000","last_became_active":"2024-09-17T23:46:41.367488+0000","last_became_peered":"2024-09-17T23:46:41.367488+0000","last_unstale":"2024-09-18T01:17:21.377936+0000","last_undegraded":"2024-09-18T01:17:21.377936+0000","last_fullsized":"2024-09-18T01:17:21.377936+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:17:21.377741+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:17:21.377741+0000","last_clean_scrub_stamp":"2024-09-18T01:17:21.377741+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T11:46:05.380525+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.0011158380000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":4009,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:20.403953+0000","last_change":"2024-09-18T01:17:20.403979+0000","last_active":"2024-09-18T01:17:20.403953+0000","last_peered":"2024-09-18T01:17:20.403953+0000","last_clean":"2024-09-18T01:17:20.403953+0000","last_became_active":"2024-09-17T23:46:41.365093+0000","last_became_peered":"2024-09-17T23:46:41.365093+0000","last_unstale":"2024-09-18T01:17:20.403953+0000","last_undegraded":"2024-09-18T01:17:20.403953+0000","last_fullsized":"2024-09-18T01:17:20.403953+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:17:20.403853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:17:20.403853+0000","last_clean_scrub_stamp":"2024-09-18T01:17:20.403853+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T04:22:03.757602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011454340000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"1985'2","reported_seq":4009,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:21.572210+0000","last_change":"2024-09-18T01:17:21.572283+0000","last_active":"2024-09-18T01:17:21.572210+0000","last_peered":"2024-09-18T01:17:21.572210+0000","last_clean":"2024-09-18T01:17:21.572210+0000","last_became_active":"2024-09-17T23:46:41.364533+0000","last_became_peered":"2024-09-17T23:46:41.364533+0000","last_unstale":"2024-09-18T01:17:21.572210+0000","last_undegraded":"2024-09-18T01:17:21.572210+0000","last_fullsized":"2024-09-18T01:17:21.572210+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"1985'2","last_scrub_stamp":"2024-09-18T01:17:21.572013+0000","last_deep_scrub":"1985'2","last_deep_scrub_stamp":"2024-09-18T01:17:21.572013+0000","last_clean_scrub_stamp":"2024-09-18T01:17:21.572013+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:20:07.492658+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.00077234799999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"1985'127","reported_seq":4173,"reported_epoch":1985,"state":"active+clean","last_fresh":"2024-09-18T01:17:20.565689+0000","last_change":"2024-09-18T01:17:20.565740+0000","last_active":"2024-09-18T01:17:20.565689+0000","last_peered":"2024-09-18T01:17:20.565689+0000","last_clean":"2024-09-18T01:17:20.565689+0000","last_became_active":"2024-09-17T23:46:37.587086+0000","last_became_peered":"2024-09-17T23:46:37.587086+0000","last_unstale":"2024-09-18T01:17:20.565689+0000","last_undegraded":"2024-09-18T01:17:20.565689+0000","last_fullsized":"2024-09-18T01:17:20.565689+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"1985'127","last_scrub_stamp":"2024-09-18T01:17:20.565582+0000","last_deep_scrub":"1985'127","last_deep_scrub_stamp":"2024-09-18T01:17:20.565582+0000","last_clean_scrub_stamp":"2024-09-18T01:17:20.565582+0000","objects_scrubbed":2,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:20:02.936629+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127,"ondisk_log_size":127,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706764,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1024836,"kb_used_data":5364,"kb_used_omap":1,"kb_used_meta":1019454,"kb_avail":93347004,"statfs":{"total":96636764160,"available":95587332096,"internally_reserved":0,"allocated":5492736,"data_stored":3044930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1043921349},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654706760,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":997644,"kb_used_data":5948,"kb_used_omap":1,"kb_used_meta":991678,"kb_avail":93374196,"statfs":{"total":96636764160,"available":95615176704,"internally_reserved":0,"allocated":6090752,"data_stored":3635317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1015478725},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706764,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1128268,"kb_used_data":5948,"kb_used_omap":1,"kb_used_meta":1122302,"kb_avail":93243572,"statfs":{"total":96636764160,"available":95481417728,"internally_reserved":0,"allocated":6090752,"data_stored":3635317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1149237701},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:17:41.009 DEBUG:teuthology.orchestra.run.smithi049:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2024-09-18T01:17:41.333 INFO:teuthology.misc:Shutting down mds daemons... 2024-09-18T01:17:41.333 INFO:teuthology.misc:Shutting down osd daemons... 2024-09-18T01:17:41.333 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2024-09-18T01:17:41.334 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:17:44.230 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~0s 2024-09-18T01:17:47.435 INFO:tasks.ceph.cluster2.osd.0:Stopped 2024-09-18T01:17:47.435 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2024-09-18T01:17:47.436 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:17:49.533 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.1 is failed for ~0s 2024-09-18T01:17:49.534 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 has been restored 2024-09-18T01:17:53.537 INFO:tasks.ceph.cluster2.osd.1:Stopped 2024-09-18T01:17:53.537 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2024-09-18T01:17:53.538 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:17:54.736 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 is failed for ~0s 2024-09-18T01:17:54.736 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.1 has been restored 2024-09-18T01:17:59.639 INFO:tasks.ceph.cluster2.osd.2:Stopped 2024-09-18T01:17:59.639 INFO:teuthology.misc:Shutting down mgr daemons... 2024-09-18T01:17:59.640 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2024-09-18T01:17:59.640 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:17:59.697 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2024-09-18T01:17:59.697 INFO:teuthology.misc:Shutting down mon daemons... 2024-09-18T01:17:59.697 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2024-09-18T01:17:59.697 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:17:59.731 INFO:tasks.ceph.cluster2.mon.a:Stopped 2024-09-18T01:17:59.731 INFO:tasks.ceph:Checking cluster log for badness... 2024-09-18T01:17:59.731 DEBUG:teuthology.orchestra.run.smithi049:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2024-09-18T01:17:59.737 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 has been restored 2024-09-18T01:17:59.857 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@smithi049.front.sepia.ceph.com 2024-09-18T01:17:59.857 DEBUG:teuthology.orchestra.run.smithi049:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2024-09-18T01:18:00.199 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@smithi049.front.sepia.ceph.com 2024-09-18T01:18:00.199 DEBUG:teuthology.orchestra.run.smithi049:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2024-09-18T01:18:00.423 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@smithi049.front.sepia.ceph.com 2024-09-18T01:18:00.424 DEBUG:teuthology.orchestra.run.smithi049:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2024-09-18T01:18:00.641 INFO:tasks.ceph:Archiving mon data... 2024-09-18T01:18:00.643 DEBUG:teuthology.misc:Transferring archived files from smithi049:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601/data/cluster2.mon.a.tgz 2024-09-18T01:18:00.643 DEBUG:teuthology.orchestra.run.smithi049:> mktemp 2024-09-18T01:18:00.648 INFO:teuthology.orchestra.run.smithi049.stdout:/tmp/tmp.hG4FhklK4b 2024-09-18T01:18:00.648 DEBUG:teuthology.orchestra.run.smithi049:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.hG4FhklK4b 2024-09-18T01:18:00.829 DEBUG:teuthology.orchestra.run.smithi049:> sudo chmod 0666 /tmp/tmp.hG4FhklK4b 2024-09-18T01:18:00.936 DEBUG:teuthology.orchestra.remote:smithi049:/tmp/tmp.hG4FhklK4b is 552KB 2024-09-18T01:18:00.999 DEBUG:teuthology.orchestra.run.smithi049:> rm -fr /tmp/tmp.hG4FhklK4b 2024-09-18T01:18:01.004 INFO:tasks.ceph:Cleaning ceph cluster... 2024-09-18T01:18:01.004 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2024-09-18T01:18:01.007 DEBUG:teuthology.orchestra.run.smithi049:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2024-09-18T01:18:01.219 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-09-18T01:18:01.229 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2024-09-18T01:18:01.229 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-18T01:18:01.822 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-18T01:18:01.823 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-09-18T01:18:01.871 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":4770,"stamp":"2024-09-18T01:18:01.772952+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":171,"num_write_kb":2550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4565660,"kb_used_data":14176,"kb_used_omap":4,"kb_used_meta":4551419,"kb_avail":278549860,"statfs":{"total":289910292480,"available":285235056640,"internally_reserved":0,"allocated":14516224,"data_stored":8515438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":4660653391},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003077"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":4053,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.850867+0000","last_change":"2024-09-17T23:46:05.734682+0000","last_active":"2024-09-18T01:17:12.850867+0000","last_peered":"2024-09-18T01:17:12.850867+0000","last_clean":"2024-09-18T01:17:12.850867+0000","last_became_active":"2024-09-17T23:46:03.734118+0000","last_became_peered":"2024-09-17T23:46:03.734118+0000","last_unstale":"2024-09-18T01:17:12.850867+0000","last_undegraded":"2024-09-18T01:17:12.850867+0000","last_fullsized":"2024-09-18T01:17:12.850867+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:58:19.460625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002359271,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":4053,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.850192+0000","last_change":"2024-09-17T23:46:05.734204+0000","last_active":"2024-09-18T01:17:12.850192+0000","last_peered":"2024-09-18T01:17:12.850192+0000","last_clean":"2024-09-18T01:17:12.850192+0000","last_became_active":"2024-09-17T23:46:03.734798+0000","last_became_peered":"2024-09-17T23:46:03.734798+0000","last_unstale":"2024-09-18T01:17:12.850192+0000","last_undegraded":"2024-09-18T01:17:12.850192+0000","last_fullsized":"2024-09-18T01:17:12.850192+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T00:38:24.564262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018543850000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":4054,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.851639+0000","last_change":"2024-09-17T23:46:05.734026+0000","last_active":"2024-09-18T01:17:12.851639+0000","last_peered":"2024-09-18T01:17:12.851639+0000","last_clean":"2024-09-18T01:17:12.851639+0000","last_became_active":"2024-09-17T23:46:03.733171+0000","last_became_peered":"2024-09-17T23:46:03.733171+0000","last_unstale":"2024-09-18T01:17:12.851639+0000","last_undegraded":"2024-09-18T01:17:12.851639+0000","last_fullsized":"2024-09-18T01:17:12.851639+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:02:46.037354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016983600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":4055,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.846789+0000","last_change":"2024-09-17T23:46:05.733455+0000","last_active":"2024-09-18T01:17:12.846789+0000","last_peered":"2024-09-18T01:17:12.846789+0000","last_clean":"2024-09-18T01:17:12.846789+0000","last_became_active":"2024-09-17T23:46:03.733266+0000","last_became_peered":"2024-09-17T23:46:03.733266+0000","last_unstale":"2024-09-18T01:17:12.846789+0000","last_undegraded":"2024-09-18T01:17:12.846789+0000","last_fullsized":"2024-09-18T01:17:12.846789+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:46:17.826212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001165857,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":4056,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.026002+0000","last_change":"2024-09-17T23:46:05.739829+0000","last_active":"2024-09-18T01:17:12.026002+0000","last_peered":"2024-09-18T01:17:12.026002+0000","last_clean":"2024-09-18T01:17:12.026002+0000","last_became_active":"2024-09-17T23:46:03.738379+0000","last_became_peered":"2024-09-17T23:46:03.738379+0000","last_unstale":"2024-09-18T01:17:12.026002+0000","last_undegraded":"2024-09-18T01:17:12.026002+0000","last_fullsized":"2024-09-18T01:17:12.026002+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:26:27.164769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019871049999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":4053,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.868184+0000","last_change":"2024-09-17T23:46:06.244182+0000","last_active":"2024-09-18T01:17:12.868184+0000","last_peered":"2024-09-18T01:17:12.868184+0000","last_clean":"2024-09-18T01:17:12.868184+0000","last_became_active":"2024-09-17T23:46:03.730771+0000","last_became_peered":"2024-09-17T23:46:03.730771+0000","last_unstale":"2024-09-18T01:17:12.868184+0000","last_undegraded":"2024-09-18T01:17:12.868184+0000","last_fullsized":"2024-09-18T01:17:12.868184+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:32:53.624364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040605900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":4054,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.865863+0000","last_change":"2024-09-17T23:46:06.243995+0000","last_active":"2024-09-18T01:17:12.865863+0000","last_peered":"2024-09-18T01:17:12.865863+0000","last_clean":"2024-09-18T01:17:12.865863+0000","last_became_active":"2024-09-17T23:46:03.731476+0000","last_became_peered":"2024-09-17T23:46:03.731476+0000","last_unstale":"2024-09-18T01:17:12.865863+0000","last_undegraded":"2024-09-18T01:17:12.865863+0000","last_fullsized":"2024-09-18T01:17:12.865863+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:54:18.516230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031242999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":4056,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.852020+0000","last_change":"2024-09-17T23:46:05.733399+0000","last_active":"2024-09-18T01:17:12.852020+0000","last_peered":"2024-09-18T01:17:12.852020+0000","last_clean":"2024-09-18T01:17:12.852020+0000","last_became_active":"2024-09-17T23:46:03.729614+0000","last_became_peered":"2024-09-17T23:46:03.729614+0000","last_unstale":"2024-09-18T01:17:12.852020+0000","last_undegraded":"2024-09-18T01:17:12.852020+0000","last_fullsized":"2024-09-18T01:17:12.852020+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:19:46.872257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011240410000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"432'125","reported_seq":4218,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.850084+0000","last_change":"2024-09-17T23:45:59.372129+0000","last_active":"2024-09-18T01:17:12.850084+0000","last_peered":"2024-09-18T01:17:12.850084+0000","last_clean":"2024-09-18T01:17:12.850084+0000","last_became_active":"2024-09-17T23:45:59.369113+0000","last_became_peered":"2024-09-17T23:45:59.369113+0000","last_unstale":"2024-09-18T01:17:12.850084+0000","last_undegraded":"2024-09-18T01:17:12.850084+0000","last_fullsized":"2024-09-18T01:17:12.850084+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:45:58.030243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:45:58.030243+0000","last_clean_scrub_stamp":"2024-09-17T23:45:58.030243+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:08:20.028364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125,"ondisk_log_size":125,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359739481,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1379204,"kb_used_data":4336,"kb_used_omap":1,"kb_used_meta":1374846,"kb_avail":92992636,"statfs":{"total":96636764160,"available":95224459264,"internally_reserved":0,"allocated":4440064,"data_stored":2444888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1407842757},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359739478,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1520268,"kb_used_data":4920,"kb_used_omap":1,"kb_used_meta":1515326,"kb_avail":92851572,"statfs":{"total":96636764160,"available":95080009728,"internally_reserved":0,"allocated":5038080,"data_stored":3035275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1551694277},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359739477,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1666188,"kb_used_data":4920,"kb_used_omap":1,"kb_used_meta":1661246,"kb_avail":92705652,"statfs":{"total":96636764160,"available":94930587648,"internally_reserved":0,"allocated":5038080,"data_stored":3035275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1701116357},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:18:01.872 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-18T01:18:02.176 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-18T01:18:02.176 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-09-18T01:18:02.193 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":4770,"stamp":"2024-09-18T01:18:01.772952+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":171,"num_write_kb":2550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4565660,"kb_used_data":14176,"kb_used_omap":4,"kb_used_meta":4551419,"kb_avail":278549860,"statfs":{"total":289910292480,"available":285235056640,"internally_reserved":0,"allocated":14516224,"data_stored":8515438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":4660653391},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003077"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":4053,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.850867+0000","last_change":"2024-09-17T23:46:05.734682+0000","last_active":"2024-09-18T01:17:12.850867+0000","last_peered":"2024-09-18T01:17:12.850867+0000","last_clean":"2024-09-18T01:17:12.850867+0000","last_became_active":"2024-09-17T23:46:03.734118+0000","last_became_peered":"2024-09-17T23:46:03.734118+0000","last_unstale":"2024-09-18T01:17:12.850867+0000","last_undegraded":"2024-09-18T01:17:12.850867+0000","last_fullsized":"2024-09-18T01:17:12.850867+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:58:19.460625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002359271,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":4053,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.850192+0000","last_change":"2024-09-17T23:46:05.734204+0000","last_active":"2024-09-18T01:17:12.850192+0000","last_peered":"2024-09-18T01:17:12.850192+0000","last_clean":"2024-09-18T01:17:12.850192+0000","last_became_active":"2024-09-17T23:46:03.734798+0000","last_became_peered":"2024-09-17T23:46:03.734798+0000","last_unstale":"2024-09-18T01:17:12.850192+0000","last_undegraded":"2024-09-18T01:17:12.850192+0000","last_fullsized":"2024-09-18T01:17:12.850192+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T00:38:24.564262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018543850000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":4054,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.851639+0000","last_change":"2024-09-17T23:46:05.734026+0000","last_active":"2024-09-18T01:17:12.851639+0000","last_peered":"2024-09-18T01:17:12.851639+0000","last_clean":"2024-09-18T01:17:12.851639+0000","last_became_active":"2024-09-17T23:46:03.733171+0000","last_became_peered":"2024-09-17T23:46:03.733171+0000","last_unstale":"2024-09-18T01:17:12.851639+0000","last_undegraded":"2024-09-18T01:17:12.851639+0000","last_fullsized":"2024-09-18T01:17:12.851639+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:02:46.037354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016983600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":4055,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.846789+0000","last_change":"2024-09-17T23:46:05.733455+0000","last_active":"2024-09-18T01:17:12.846789+0000","last_peered":"2024-09-18T01:17:12.846789+0000","last_clean":"2024-09-18T01:17:12.846789+0000","last_became_active":"2024-09-17T23:46:03.733266+0000","last_became_peered":"2024-09-17T23:46:03.733266+0000","last_unstale":"2024-09-18T01:17:12.846789+0000","last_undegraded":"2024-09-18T01:17:12.846789+0000","last_fullsized":"2024-09-18T01:17:12.846789+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:46:17.826212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001165857,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":4056,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.026002+0000","last_change":"2024-09-17T23:46:05.739829+0000","last_active":"2024-09-18T01:17:12.026002+0000","last_peered":"2024-09-18T01:17:12.026002+0000","last_clean":"2024-09-18T01:17:12.026002+0000","last_became_active":"2024-09-17T23:46:03.738379+0000","last_became_peered":"2024-09-17T23:46:03.738379+0000","last_unstale":"2024-09-18T01:17:12.026002+0000","last_undegraded":"2024-09-18T01:17:12.026002+0000","last_fullsized":"2024-09-18T01:17:12.026002+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:26:27.164769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019871049999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":4053,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.868184+0000","last_change":"2024-09-17T23:46:06.244182+0000","last_active":"2024-09-18T01:17:12.868184+0000","last_peered":"2024-09-18T01:17:12.868184+0000","last_clean":"2024-09-18T01:17:12.868184+0000","last_became_active":"2024-09-17T23:46:03.730771+0000","last_became_peered":"2024-09-17T23:46:03.730771+0000","last_unstale":"2024-09-18T01:17:12.868184+0000","last_undegraded":"2024-09-18T01:17:12.868184+0000","last_fullsized":"2024-09-18T01:17:12.868184+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:32:53.624364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040605900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":4054,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.865863+0000","last_change":"2024-09-17T23:46:06.243995+0000","last_active":"2024-09-18T01:17:12.865863+0000","last_peered":"2024-09-18T01:17:12.865863+0000","last_clean":"2024-09-18T01:17:12.865863+0000","last_became_active":"2024-09-17T23:46:03.731476+0000","last_became_peered":"2024-09-17T23:46:03.731476+0000","last_unstale":"2024-09-18T01:17:12.865863+0000","last_undegraded":"2024-09-18T01:17:12.865863+0000","last_fullsized":"2024-09-18T01:17:12.865863+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:54:18.516230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031242999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":4056,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.852020+0000","last_change":"2024-09-17T23:46:05.733399+0000","last_active":"2024-09-18T01:17:12.852020+0000","last_peered":"2024-09-18T01:17:12.852020+0000","last_clean":"2024-09-18T01:17:12.852020+0000","last_became_active":"2024-09-17T23:46:03.729614+0000","last_became_peered":"2024-09-17T23:46:03.729614+0000","last_unstale":"2024-09-18T01:17:12.852020+0000","last_undegraded":"2024-09-18T01:17:12.852020+0000","last_fullsized":"2024-09-18T01:17:12.852020+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:19:46.872257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011240410000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"432'125","reported_seq":4218,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.850084+0000","last_change":"2024-09-17T23:45:59.372129+0000","last_active":"2024-09-18T01:17:12.850084+0000","last_peered":"2024-09-18T01:17:12.850084+0000","last_clean":"2024-09-18T01:17:12.850084+0000","last_became_active":"2024-09-17T23:45:59.369113+0000","last_became_peered":"2024-09-17T23:45:59.369113+0000","last_unstale":"2024-09-18T01:17:12.850084+0000","last_undegraded":"2024-09-18T01:17:12.850084+0000","last_fullsized":"2024-09-18T01:17:12.850084+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:45:58.030243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:45:58.030243+0000","last_clean_scrub_stamp":"2024-09-17T23:45:58.030243+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:08:20.028364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125,"ondisk_log_size":125,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359739481,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1379204,"kb_used_data":4336,"kb_used_omap":1,"kb_used_meta":1374846,"kb_avail":92992636,"statfs":{"total":96636764160,"available":95224459264,"internally_reserved":0,"allocated":4440064,"data_stored":2444888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1407842757},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359739478,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1520268,"kb_used_data":4920,"kb_used_omap":1,"kb_used_meta":1515326,"kb_avail":92851572,"statfs":{"total":96636764160,"available":95080009728,"internally_reserved":0,"allocated":5038080,"data_stored":3035275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1551694277},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359739477,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1666188,"kb_used_data":4920,"kb_used_omap":1,"kb_used_meta":1661246,"kb_avail":92705652,"statfs":{"total":96636764160,"available":94930587648,"internally_reserved":0,"allocated":5038080,"data_stored":3035275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1701116357},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:18:02.194 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2024-09-18T01:18:02.194 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-18T01:18:02.479 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-18T01:18:02.479 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-09-18T01:18:02.496 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":4770,"stamp":"2024-09-18T01:18:01.772952+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":171,"num_write_kb":2550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4565660,"kb_used_data":14176,"kb_used_omap":4,"kb_used_meta":4551419,"kb_avail":278549860,"statfs":{"total":289910292480,"available":285235056640,"internally_reserved":0,"allocated":14516224,"data_stored":8515438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":4660653391},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003077"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":4053,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.850867+0000","last_change":"2024-09-17T23:46:05.734682+0000","last_active":"2024-09-18T01:17:12.850867+0000","last_peered":"2024-09-18T01:17:12.850867+0000","last_clean":"2024-09-18T01:17:12.850867+0000","last_became_active":"2024-09-17T23:46:03.734118+0000","last_became_peered":"2024-09-17T23:46:03.734118+0000","last_unstale":"2024-09-18T01:17:12.850867+0000","last_undegraded":"2024-09-18T01:17:12.850867+0000","last_fullsized":"2024-09-18T01:17:12.850867+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:58:19.460625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002359271,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":4053,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.850192+0000","last_change":"2024-09-17T23:46:05.734204+0000","last_active":"2024-09-18T01:17:12.850192+0000","last_peered":"2024-09-18T01:17:12.850192+0000","last_clean":"2024-09-18T01:17:12.850192+0000","last_became_active":"2024-09-17T23:46:03.734798+0000","last_became_peered":"2024-09-17T23:46:03.734798+0000","last_unstale":"2024-09-18T01:17:12.850192+0000","last_undegraded":"2024-09-18T01:17:12.850192+0000","last_fullsized":"2024-09-18T01:17:12.850192+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T00:38:24.564262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018543850000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":4054,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.851639+0000","last_change":"2024-09-17T23:46:05.734026+0000","last_active":"2024-09-18T01:17:12.851639+0000","last_peered":"2024-09-18T01:17:12.851639+0000","last_clean":"2024-09-18T01:17:12.851639+0000","last_became_active":"2024-09-17T23:46:03.733171+0000","last_became_peered":"2024-09-17T23:46:03.733171+0000","last_unstale":"2024-09-18T01:17:12.851639+0000","last_undegraded":"2024-09-18T01:17:12.851639+0000","last_fullsized":"2024-09-18T01:17:12.851639+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:02:46.037354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016983600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":4055,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.846789+0000","last_change":"2024-09-17T23:46:05.733455+0000","last_active":"2024-09-18T01:17:12.846789+0000","last_peered":"2024-09-18T01:17:12.846789+0000","last_clean":"2024-09-18T01:17:12.846789+0000","last_became_active":"2024-09-17T23:46:03.733266+0000","last_became_peered":"2024-09-17T23:46:03.733266+0000","last_unstale":"2024-09-18T01:17:12.846789+0000","last_undegraded":"2024-09-18T01:17:12.846789+0000","last_fullsized":"2024-09-18T01:17:12.846789+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:46:17.826212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001165857,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":4056,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.026002+0000","last_change":"2024-09-17T23:46:05.739829+0000","last_active":"2024-09-18T01:17:12.026002+0000","last_peered":"2024-09-18T01:17:12.026002+0000","last_clean":"2024-09-18T01:17:12.026002+0000","last_became_active":"2024-09-17T23:46:03.738379+0000","last_became_peered":"2024-09-17T23:46:03.738379+0000","last_unstale":"2024-09-18T01:17:12.026002+0000","last_undegraded":"2024-09-18T01:17:12.026002+0000","last_fullsized":"2024-09-18T01:17:12.026002+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:26:27.164769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019871049999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":4053,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.868184+0000","last_change":"2024-09-17T23:46:06.244182+0000","last_active":"2024-09-18T01:17:12.868184+0000","last_peered":"2024-09-18T01:17:12.868184+0000","last_clean":"2024-09-18T01:17:12.868184+0000","last_became_active":"2024-09-17T23:46:03.730771+0000","last_became_peered":"2024-09-17T23:46:03.730771+0000","last_unstale":"2024-09-18T01:17:12.868184+0000","last_undegraded":"2024-09-18T01:17:12.868184+0000","last_fullsized":"2024-09-18T01:17:12.868184+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:32:53.624364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040605900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":4054,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.865863+0000","last_change":"2024-09-17T23:46:06.243995+0000","last_active":"2024-09-18T01:17:12.865863+0000","last_peered":"2024-09-18T01:17:12.865863+0000","last_clean":"2024-09-18T01:17:12.865863+0000","last_became_active":"2024-09-17T23:46:03.731476+0000","last_became_peered":"2024-09-17T23:46:03.731476+0000","last_unstale":"2024-09-18T01:17:12.865863+0000","last_undegraded":"2024-09-18T01:17:12.865863+0000","last_fullsized":"2024-09-18T01:17:12.865863+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:54:18.516230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031242999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":4056,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.852020+0000","last_change":"2024-09-17T23:46:05.733399+0000","last_active":"2024-09-18T01:17:12.852020+0000","last_peered":"2024-09-18T01:17:12.852020+0000","last_clean":"2024-09-18T01:17:12.852020+0000","last_became_active":"2024-09-17T23:46:03.729614+0000","last_became_peered":"2024-09-17T23:46:03.729614+0000","last_unstale":"2024-09-18T01:17:12.852020+0000","last_undegraded":"2024-09-18T01:17:12.852020+0000","last_fullsized":"2024-09-18T01:17:12.852020+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:19:46.872257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011240410000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"432'125","reported_seq":4218,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.850084+0000","last_change":"2024-09-17T23:45:59.372129+0000","last_active":"2024-09-18T01:17:12.850084+0000","last_peered":"2024-09-18T01:17:12.850084+0000","last_clean":"2024-09-18T01:17:12.850084+0000","last_became_active":"2024-09-17T23:45:59.369113+0000","last_became_peered":"2024-09-17T23:45:59.369113+0000","last_unstale":"2024-09-18T01:17:12.850084+0000","last_undegraded":"2024-09-18T01:17:12.850084+0000","last_fullsized":"2024-09-18T01:17:12.850084+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:45:58.030243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:45:58.030243+0000","last_clean_scrub_stamp":"2024-09-17T23:45:58.030243+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:08:20.028364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125,"ondisk_log_size":125,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359739481,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1379204,"kb_used_data":4336,"kb_used_omap":1,"kb_used_meta":1374846,"kb_avail":92992636,"statfs":{"total":96636764160,"available":95224459264,"internally_reserved":0,"allocated":4440064,"data_stored":2444888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1407842757},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359739478,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1520268,"kb_used_data":4920,"kb_used_omap":1,"kb_used_meta":1515326,"kb_avail":92851572,"statfs":{"total":96636764160,"available":95080009728,"internally_reserved":0,"allocated":5038080,"data_stored":3035275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1551694277},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359739477,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1666188,"kb_used_data":4920,"kb_used_omap":1,"kb_used_meta":1661246,"kb_avail":92705652,"statfs":{"total":96636764160,"available":94930587648,"internally_reserved":0,"allocated":5038080,"data_stored":3035275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1701116357},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:18:02.497 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-18T01:18:02.784 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-18T01:18:02.784 INFO:teuthology.orchestra.run.smithi028.stdout:{"epoch":2011,"fsid":"57e7c284-39b0-4b91-954c-a87a8b8ca988","created":"2024-09-17T23:45:46.808870+0000","modified":"2024-09-18T01:17:12.010175+0000","last_up_change":"2024-09-17T23:45:57.028864+0000","last_in_change":"2024-09-17T23:45:47.715296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T23:45:57.757216+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-17T23:46:02.187572+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"8e6f69c1-27ba-4644-9817-26dce7dd505c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6816","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6817","nonce":2136955652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6818","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6819","nonce":2136955652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6822","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6823","nonce":2136955652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6820","nonce":2136955652},{"type":"v1","addr":"172.21.15.28:6821","nonce":2136955652}]},"public_addr":"172.21.15.28:6817/2136955652","cluster_addr":"172.21.15.28:6819/2136955652","heartbeat_back_addr":"172.21.15.28:6823/2136955652","heartbeat_front_addr":"172.21.15.28:6821/2136955652","state":["exists","up"]},{"osd":1,"uuid":"9bff203c-5eea-4198-b867-3161de2c53d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6808","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6809","nonce":3505522461}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6810","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6811","nonce":3505522461}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6814","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6815","nonce":3505522461}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6812","nonce":3505522461},{"type":"v1","addr":"172.21.15.28:6813","nonce":3505522461}]},"public_addr":"172.21.15.28:6809/3505522461","cluster_addr":"172.21.15.28:6811/3505522461","heartbeat_back_addr":"172.21.15.28:6815/3505522461","heartbeat_front_addr":"172.21.15.28:6813/3505522461","state":["exists","up"]},{"osd":2,"uuid":"6fdba670-6e7a-4175-82f1-097f630f6148","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6800","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6801","nonce":182730647}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6802","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6803","nonce":182730647}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6806","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6807","nonce":182730647}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.28:6804","nonce":182730647},{"type":"v1","addr":"172.21.15.28:6805","nonce":182730647}]},"public_addr":"172.21.15.28:6801/182730647","cluster_addr":"172.21.15.28:6803/182730647","heartbeat_back_addr":"172.21.15.28:6807/182730647","heartbeat_front_addr":"172.21.15.28:6805/182730647","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:45:54.289133+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:45:54.275284+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T23:45:54.091032+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":3282,"length":1},{"begin":3295,"length":1},{"begin":3302,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T01:18:03.804 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2024-09-18T01:18:03.804 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-09-18T01:18:03.971 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-18T01:18:03.971 INFO:teuthology.orchestra.run.smithi028.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-18T01:18:03.971 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-18T01:18:03.987 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2024-09-18T01:18:04.280 INFO:teuthology.orchestra.run.smithi028.stderr:instructed osd(s) 0 to deep-scrub 2024-09-18T01:18:04.298 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2024-09-18T01:18:04.298 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-09-18T01:18:04.451 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-18T01:18:04.452 INFO:teuthology.orchestra.run.smithi028.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-18T01:18:04.452 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-18T01:18:04.468 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2024-09-18T01:18:04.772 INFO:teuthology.orchestra.run.smithi028.stderr:instructed osd(s) 1 to deep-scrub 2024-09-18T01:18:04.792 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2024-09-18T01:18:04.792 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-09-18T01:18:04.936 INFO:teuthology.orchestra.run.smithi028.stdout:{ 2024-09-18T01:18:04.936 INFO:teuthology.orchestra.run.smithi028.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-18T01:18:04.937 INFO:teuthology.orchestra.run.smithi028.stdout:} 2024-09-18T01:18:04.951 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2024-09-18T01:18:05.237 INFO:teuthology.orchestra.run.smithi028.stderr:instructed osd(s) 2 to deep-scrub 2024-09-18T01:18:05.255 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-18T01:18:05.544 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-18T01:18:05.544 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-09-18T01:18:05.561 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":4771,"stamp":"2024-09-18T01:18:03.773450+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":171,"num_write_kb":2550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4565660,"kb_used_data":14176,"kb_used_omap":4,"kb_used_meta":4551419,"kb_avail":278549860,"statfs":{"total":289910292480,"available":285235056640,"internally_reserved":0,"allocated":14516224,"data_stored":8515438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":4660653391},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003023"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":4053,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.850867+0000","last_change":"2024-09-17T23:46:05.734682+0000","last_active":"2024-09-18T01:17:12.850867+0000","last_peered":"2024-09-18T01:17:12.850867+0000","last_clean":"2024-09-18T01:17:12.850867+0000","last_became_active":"2024-09-17T23:46:03.734118+0000","last_became_peered":"2024-09-17T23:46:03.734118+0000","last_unstale":"2024-09-18T01:17:12.850867+0000","last_undegraded":"2024-09-18T01:17:12.850867+0000","last_fullsized":"2024-09-18T01:17:12.850867+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:58:19.460625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002359271,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":4053,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.850192+0000","last_change":"2024-09-17T23:46:05.734204+0000","last_active":"2024-09-18T01:17:12.850192+0000","last_peered":"2024-09-18T01:17:12.850192+0000","last_clean":"2024-09-18T01:17:12.850192+0000","last_became_active":"2024-09-17T23:46:03.734798+0000","last_became_peered":"2024-09-17T23:46:03.734798+0000","last_unstale":"2024-09-18T01:17:12.850192+0000","last_undegraded":"2024-09-18T01:17:12.850192+0000","last_fullsized":"2024-09-18T01:17:12.850192+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T00:38:24.564262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018543850000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":4054,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.851639+0000","last_change":"2024-09-17T23:46:05.734026+0000","last_active":"2024-09-18T01:17:12.851639+0000","last_peered":"2024-09-18T01:17:12.851639+0000","last_clean":"2024-09-18T01:17:12.851639+0000","last_became_active":"2024-09-17T23:46:03.733171+0000","last_became_peered":"2024-09-17T23:46:03.733171+0000","last_unstale":"2024-09-18T01:17:12.851639+0000","last_undegraded":"2024-09-18T01:17:12.851639+0000","last_fullsized":"2024-09-18T01:17:12.851639+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:02:46.037354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016983600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":4055,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.846789+0000","last_change":"2024-09-17T23:46:05.733455+0000","last_active":"2024-09-18T01:17:12.846789+0000","last_peered":"2024-09-18T01:17:12.846789+0000","last_clean":"2024-09-18T01:17:12.846789+0000","last_became_active":"2024-09-17T23:46:03.733266+0000","last_became_peered":"2024-09-17T23:46:03.733266+0000","last_unstale":"2024-09-18T01:17:12.846789+0000","last_undegraded":"2024-09-18T01:17:12.846789+0000","last_fullsized":"2024-09-18T01:17:12.846789+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:46:17.826212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001165857,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":4056,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.026002+0000","last_change":"2024-09-17T23:46:05.739829+0000","last_active":"2024-09-18T01:17:12.026002+0000","last_peered":"2024-09-18T01:17:12.026002+0000","last_clean":"2024-09-18T01:17:12.026002+0000","last_became_active":"2024-09-17T23:46:03.738379+0000","last_became_peered":"2024-09-17T23:46:03.738379+0000","last_unstale":"2024-09-18T01:17:12.026002+0000","last_undegraded":"2024-09-18T01:17:12.026002+0000","last_fullsized":"2024-09-18T01:17:12.026002+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:26:27.164769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0019871049999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":4053,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.868184+0000","last_change":"2024-09-17T23:46:06.244182+0000","last_active":"2024-09-18T01:17:12.868184+0000","last_peered":"2024-09-18T01:17:12.868184+0000","last_clean":"2024-09-18T01:17:12.868184+0000","last_became_active":"2024-09-17T23:46:03.730771+0000","last_became_peered":"2024-09-17T23:46:03.730771+0000","last_unstale":"2024-09-18T01:17:12.868184+0000","last_undegraded":"2024-09-18T01:17:12.868184+0000","last_fullsized":"2024-09-18T01:17:12.868184+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:32:53.624364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040605900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":4054,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.865863+0000","last_change":"2024-09-17T23:46:06.243995+0000","last_active":"2024-09-18T01:17:12.865863+0000","last_peered":"2024-09-18T01:17:12.865863+0000","last_clean":"2024-09-18T01:17:12.865863+0000","last_became_active":"2024-09-17T23:46:03.731476+0000","last_became_peered":"2024-09-17T23:46:03.731476+0000","last_unstale":"2024-09-18T01:17:12.865863+0000","last_undegraded":"2024-09-18T01:17:12.865863+0000","last_fullsized":"2024-09-18T01:17:12.865863+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:54:18.516230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031242999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":4056,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.852020+0000","last_change":"2024-09-17T23:46:05.733399+0000","last_active":"2024-09-18T01:17:12.852020+0000","last_peered":"2024-09-18T01:17:12.852020+0000","last_clean":"2024-09-18T01:17:12.852020+0000","last_became_active":"2024-09-17T23:46:03.729614+0000","last_became_peered":"2024-09-17T23:46:03.729614+0000","last_unstale":"2024-09-18T01:17:12.852020+0000","last_undegraded":"2024-09-18T01:17:12.852020+0000","last_fullsized":"2024-09-18T01:17:12.852020+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:46:02.707957+0000","last_clean_scrub_stamp":"2024-09-17T23:46:02.707957+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:19:46.872257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011240410000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"432'125","reported_seq":4218,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:17:12.850084+0000","last_change":"2024-09-17T23:45:59.372129+0000","last_active":"2024-09-18T01:17:12.850084+0000","last_peered":"2024-09-18T01:17:12.850084+0000","last_clean":"2024-09-18T01:17:12.850084+0000","last_became_active":"2024-09-17T23:45:59.369113+0000","last_became_peered":"2024-09-17T23:45:59.369113+0000","last_unstale":"2024-09-18T01:17:12.850084+0000","last_undegraded":"2024-09-18T01:17:12.850084+0000","last_fullsized":"2024-09-18T01:17:12.850084+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T23:45:58.030243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T23:45:58.030243+0000","last_clean_scrub_stamp":"2024-09-17T23:45:58.030243+0000","objects_scrubbed":0,"log_size":125,"log_dups_size":0,"ondisk_log_size":125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T01:08:20.028364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125,"ondisk_log_size":125,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359739482,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1379204,"kb_used_data":4336,"kb_used_omap":1,"kb_used_meta":1374846,"kb_avail":92992636,"statfs":{"total":96636764160,"available":95224459264,"internally_reserved":0,"allocated":4440064,"data_stored":2444888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1407842757},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359739479,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1520268,"kb_used_data":4920,"kb_used_omap":1,"kb_used_meta":1515326,"kb_avail":92851572,"statfs":{"total":96636764160,"available":95080009728,"internally_reserved":0,"allocated":5038080,"data_stored":3035275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1551694277},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359739477,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1666188,"kb_used_data":4920,"kb_used_omap":1,"kb_used_meta":1661246,"kb_avail":92705652,"statfs":{"total":96636764160,"available":94930587648,"internally_reserved":0,"allocated":5038080,"data_stored":3035275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1701116357},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:18:05.563 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2024-09-17T23:46:02.707957+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=2, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=18, tm_sec=2, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:18:05.564 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2024-09-17T23:46:02.707957+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=2, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=18, tm_sec=2, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:18:05.564 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2024-09-17T23:46:02.707957+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=2, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=18, tm_sec=2, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:18:05.564 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2024-09-17T23:46:02.707957+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=2, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=18, tm_sec=2, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:18:05.564 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2024-09-17T23:46:02.707957+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=2, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=18, tm_sec=2, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:18:05.565 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2024-09-17T23:46:02.707957+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=2, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=18, tm_sec=2, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:18:05.565 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2024-09-17T23:46:02.707957+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=2, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=18, tm_sec=2, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:18:05.565 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2024-09-17T23:46:02.707957+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=46, tm_sec=2, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=18, tm_sec=2, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:18:05.565 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-09-17T23:45:58.030243+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=17, tm_hour=23, tm_min=45, tm_sec=58, tm_wday=1, tm_yday=261, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=18, tm_sec=2, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:18:05.566 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-09-18T01:18:25.567 DEBUG:teuthology.orchestra.run.smithi028:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-18T01:18:25.877 INFO:teuthology.orchestra.run.smithi028.stdout: 2024-09-18T01:18:25.878 INFO:teuthology.orchestra.run.smithi028.stderr:dumped all 2024-09-18T01:18:25.897 INFO:teuthology.orchestra.run.smithi028.stdout:{"pg_ready":true,"pg_map":{"version":4782,"stamp":"2024-09-18T01:18:25.779006+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":171,"num_write_kb":2550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132,"ondisk_log_size":132,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4565648,"kb_used_data":14176,"kb_used_omap":4,"kb_used_meta":4551419,"kb_avail":278549872,"statfs":{"total":289910292480,"available":285235068928,"internally_reserved":0,"allocated":14516224,"data_stored":8515438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":4660653391},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002984"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":4062,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:18:08.458733+0000","last_change":"2024-09-18T01:18:08.458792+0000","last_active":"2024-09-18T01:18:08.458733+0000","last_peered":"2024-09-18T01:18:08.458733+0000","last_clean":"2024-09-18T01:18:08.458733+0000","last_became_active":"2024-09-17T23:46:03.734118+0000","last_became_peered":"2024-09-17T23:46:03.734118+0000","last_unstale":"2024-09-18T01:18:08.458733+0000","last_undegraded":"2024-09-18T01:18:08.458733+0000","last_fullsized":"2024-09-18T01:18:08.458733+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:18:08.458519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:18:08.458519+0000","last_clean_scrub_stamp":"2024-09-18T01:18:08.458519+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T13:13:42.106594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002359271,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":4062,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:18:09.460018+0000","last_change":"2024-09-18T01:18:09.460072+0000","last_active":"2024-09-18T01:18:09.460018+0000","last_peered":"2024-09-18T01:18:09.460018+0000","last_clean":"2024-09-18T01:18:09.460018+0000","last_became_active":"2024-09-17T23:46:03.734798+0000","last_became_peered":"2024-09-17T23:46:03.734798+0000","last_unstale":"2024-09-18T01:18:09.460018+0000","last_undegraded":"2024-09-18T01:18:09.460018+0000","last_fullsized":"2024-09-18T01:18:09.460018+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:18:09.459804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:18:09.459804+0000","last_clean_scrub_stamp":"2024-09-18T01:18:09.459804+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:44:08.197243+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.0018543850000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":4064,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:18:10.426163+0000","last_change":"2024-09-18T01:18:10.426215+0000","last_active":"2024-09-18T01:18:10.426163+0000","last_peered":"2024-09-18T01:18:10.426163+0000","last_clean":"2024-09-18T01:18:10.426163+0000","last_became_active":"2024-09-17T23:46:03.733171+0000","last_became_peered":"2024-09-17T23:46:03.733171+0000","last_unstale":"2024-09-18T01:18:10.426163+0000","last_undegraded":"2024-09-18T01:18:10.426163+0000","last_fullsized":"2024-09-18T01:18:10.426163+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:18:10.425969+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:18:10.425969+0000","last_clean_scrub_stamp":"2024-09-18T01:18:10.425969+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T04:47:48.087224+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.0016983600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":4064,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:18:07.496552+0000","last_change":"2024-09-18T01:18:07.496607+0000","last_active":"2024-09-18T01:18:07.496552+0000","last_peered":"2024-09-18T01:18:07.496552+0000","last_clean":"2024-09-18T01:18:07.496552+0000","last_became_active":"2024-09-17T23:46:03.733266+0000","last_became_peered":"2024-09-17T23:46:03.733266+0000","last_unstale":"2024-09-18T01:18:07.496552+0000","last_undegraded":"2024-09-18T01:18:07.496552+0000","last_fullsized":"2024-09-18T01:18:07.496552+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:18:07.496357+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:18:07.496357+0000","last_clean_scrub_stamp":"2024-09-18T01:18:07.496357+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:53:52.757100+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.001165857,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"2011'3","reported_seq":4066,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:18:05.229500+0000","last_change":"2024-09-18T01:18:05.229590+0000","last_active":"2024-09-18T01:18:05.229500+0000","last_peered":"2024-09-18T01:18:05.229500+0000","last_clean":"2024-09-18T01:18:05.229500+0000","last_became_active":"2024-09-17T23:46:03.738379+0000","last_became_peered":"2024-09-17T23:46:03.738379+0000","last_unstale":"2024-09-18T01:18:05.229500+0000","last_undegraded":"2024-09-18T01:18:05.229500+0000","last_fullsized":"2024-09-18T01:18:05.229500+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"2011'3","last_scrub_stamp":"2024-09-18T01:18:05.229289+0000","last_deep_scrub":"2011'3","last_deep_scrub_stamp":"2024-09-18T01:18:05.229289+0000","last_clean_scrub_stamp":"2024-09-18T01:18:05.229289+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:52:58.112355+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.0019871049999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":4062,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:18:06.695549+0000","last_change":"2024-09-18T01:18:06.695643+0000","last_active":"2024-09-18T01:18:06.695549+0000","last_peered":"2024-09-18T01:18:06.695549+0000","last_clean":"2024-09-18T01:18:06.695549+0000","last_became_active":"2024-09-17T23:46:03.730771+0000","last_became_peered":"2024-09-17T23:46:03.730771+0000","last_unstale":"2024-09-18T01:18:06.695549+0000","last_undegraded":"2024-09-18T01:18:06.695549+0000","last_fullsized":"2024-09-18T01:18:06.695549+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:18:06.695316+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:18:06.695316+0000","last_clean_scrub_stamp":"2024-09-18T01:18:06.695316+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:53:00.906482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040605900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":4064,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:18:05.704599+0000","last_change":"2024-09-18T01:18:05.704599+0000","last_active":"2024-09-18T01:18:05.704599+0000","last_peered":"2024-09-18T01:18:05.704599+0000","last_clean":"2024-09-18T01:18:05.704599+0000","last_became_active":"2024-09-17T23:46:03.731476+0000","last_became_peered":"2024-09-17T23:46:03.731476+0000","last_unstale":"2024-09-18T01:18:05.704599+0000","last_undegraded":"2024-09-18T01:18:05.704599+0000","last_fullsized":"2024-09-18T01:18:05.704599+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:18:05.704306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:18:05.704306+0000","last_clean_scrub_stamp":"2024-09-18T01:18:05.704306+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T10:49:36.630457+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.00031242999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"2011'2","reported_seq":4066,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:18:06.489862+0000","last_change":"2024-09-18T01:18:06.489931+0000","last_active":"2024-09-18T01:18:06.489862+0000","last_peered":"2024-09-18T01:18:06.489862+0000","last_clean":"2024-09-18T01:18:06.489862+0000","last_became_active":"2024-09-17T23:46:03.729614+0000","last_became_peered":"2024-09-17T23:46:03.729614+0000","last_unstale":"2024-09-18T01:18:06.489862+0000","last_undegraded":"2024-09-18T01:18:06.489862+0000","last_fullsized":"2024-09-18T01:18:06.489862+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"2011'2","last_scrub_stamp":"2024-09-18T01:18:06.489676+0000","last_deep_scrub":"2011'2","last_deep_scrub_stamp":"2024-09-18T01:18:06.489676+0000","last_clean_scrub_stamp":"2024-09-18T01:18:06.489676+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T11:05:51.041989+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.0011240410000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"2011'127","reported_seq":4230,"reported_epoch":2011,"state":"active+clean","last_fresh":"2024-09-18T01:18:05.448911+0000","last_change":"2024-09-18T01:18:05.448911+0000","last_active":"2024-09-18T01:18:05.448911+0000","last_peered":"2024-09-18T01:18:05.448911+0000","last_clean":"2024-09-18T01:18:05.448911+0000","last_became_active":"2024-09-17T23:45:59.369113+0000","last_became_peered":"2024-09-17T23:45:59.369113+0000","last_unstale":"2024-09-18T01:18:05.448911+0000","last_undegraded":"2024-09-18T01:18:05.448911+0000","last_fullsized":"2024-09-18T01:18:05.448911+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"2011'127","last_scrub_stamp":"2024-09-18T01:18:05.448728+0000","last_deep_scrub":"2011'127","last_deep_scrub_stamp":"2024-09-18T01:18:05.448728+0000","last_clean_scrub_stamp":"2024-09-18T01:18:05.448728+0000","objects_scrubbed":2,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:23:45.535424+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":109,"num_write":169,"num_write_kb":2548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127,"ondisk_log_size":127,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359739486,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1379200,"kb_used_data":4336,"kb_used_omap":1,"kb_used_meta":1374846,"kb_avail":92992640,"statfs":{"total":96636764160,"available":95224463360,"internally_reserved":0,"allocated":4440064,"data_stored":2444888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1407842757},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359739483,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1520264,"kb_used_data":4920,"kb_used_omap":1,"kb_used_meta":1515326,"kb_avail":92851576,"statfs":{"total":96636764160,"available":95080013824,"internally_reserved":0,"allocated":5038080,"data_stored":3035275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1551694277},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359739482,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1666184,"kb_used_data":4920,"kb_used_omap":1,"kb_used_meta":1661246,"kb_avail":92705656,"statfs":{"total":96636764160,"available":94930591744,"internally_reserved":0,"allocated":5038080,"data_stored":3035275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1701116357},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:18:25.898 DEBUG:teuthology.orchestra.run.smithi028:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2024-09-18T01:18:26.232 INFO:teuthology.misc:Shutting down mds daemons... 2024-09-18T01:18:26.232 INFO:teuthology.misc:Shutting down osd daemons... 2024-09-18T01:18:26.233 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2024-09-18T01:18:26.233 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:18:29.200 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.0 is failed for ~0s 2024-09-18T01:18:32.335 INFO:tasks.ceph.cluster1.osd.0:Stopped 2024-09-18T01:18:32.335 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2024-09-18T01:18:32.335 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:18:34.504 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.1 is failed for ~0s 2024-09-18T01:18:34.504 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.0 has been restored 2024-09-18T01:18:38.437 INFO:tasks.ceph.cluster1.osd.1:Stopped 2024-09-18T01:18:38.438 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2024-09-18T01:18:38.438 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:18:39.707 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.2 is failed for ~0s 2024-09-18T01:18:39.708 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.1 has been restored 2024-09-18T01:18:44.539 INFO:tasks.ceph.cluster1.osd.2:Stopped 2024-09-18T01:18:44.540 INFO:teuthology.misc:Shutting down mgr daemons... 2024-09-18T01:18:44.540 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2024-09-18T01:18:44.540 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:18:44.571 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2024-09-18T01:18:44.571 INFO:teuthology.misc:Shutting down mon daemons... 2024-09-18T01:18:44.572 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2024-09-18T01:18:44.572 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:18:44.637 INFO:tasks.ceph.cluster1.mon.a:Stopped 2024-09-18T01:18:44.637 INFO:tasks.ceph:Checking cluster log for badness... 2024-09-18T01:18:44.637 DEBUG:teuthology.orchestra.run.smithi028:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2024-09-18T01:18:44.709 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.2 has been restored 2024-09-18T01:18:44.735 INFO:teuthology.orchestra.run.smithi028.stdout:2024-09-17T23:48:53.860790+0000 mon.a (mon.0) 240 : cluster [WRN] Health check failed: Degraded data redundancy: 1/2122 objects degraded (0.047%), 1 pg degraded (PG_DEGRADED) 2024-09-18T01:18:44.736 WARNING:tasks.ceph:Found errors (ERR|WRN|SEC) in cluster log 2024-09-18T01:18:44.736 DEBUG:teuthology.orchestra.run.smithi028:> sudo egrep '\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2024-09-18T01:18:44.751 DEBUG:teuthology.orchestra.run.smithi028:> sudo egrep '\[ERR\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2024-09-18T01:18:44.808 DEBUG:teuthology.orchestra.run.smithi028:> sudo egrep '\[WRN\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2024-09-18T01:18:44.865 INFO:teuthology.orchestra.run.smithi028.stdout:2024-09-17T23:48:53.860790+0000 mon.a (mon.0) 240 : cluster [WRN] Health check failed: Degraded data redundancy: 1/2122 objects degraded (0.047%), 1 pg degraded (PG_DEGRADED) 2024-09-18T01:18:44.866 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@smithi028.front.sepia.ceph.com 2024-09-18T01:18:44.867 DEBUG:teuthology.orchestra.run.smithi028:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2024-09-18T01:18:45.187 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@smithi028.front.sepia.ceph.com 2024-09-18T01:18:45.187 DEBUG:teuthology.orchestra.run.smithi028:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2024-09-18T01:18:45.584 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@smithi028.front.sepia.ceph.com 2024-09-18T01:18:45.584 DEBUG:teuthology.orchestra.run.smithi028:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2024-09-18T01:18:45.787 INFO:tasks.ceph:Archiving mon data... 2024-09-18T01:18:45.787 DEBUG:teuthology.misc:Transferring archived files from smithi028:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601/data/cluster1.mon.a.tgz 2024-09-18T01:18:45.787 DEBUG:teuthology.orchestra.run.smithi028:> mktemp 2024-09-18T01:18:45.795 INFO:teuthology.orchestra.run.smithi028.stdout:/tmp/tmp.BzPbdclFe7 2024-09-18T01:18:45.796 DEBUG:teuthology.orchestra.run.smithi028:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.BzPbdclFe7 2024-09-18T01:18:45.973 DEBUG:teuthology.orchestra.run.smithi028:> sudo chmod 0666 /tmp/tmp.BzPbdclFe7 2024-09-18T01:18:46.064 DEBUG:teuthology.orchestra.remote:smithi028:/tmp/tmp.BzPbdclFe7 is 554KB 2024-09-18T01:18:46.125 DEBUG:teuthology.orchestra.run.smithi028:> rm -fr /tmp/tmp.BzPbdclFe7 2024-09-18T01:18:46.130 INFO:tasks.ceph:Cleaning ceph cluster... 2024-09-18T01:18:46.130 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2024-09-18T01:18:46.172 DEBUG:teuthology.orchestra.run.smithi049:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2024-09-18T01:18:46.235 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-09-18T01:18:46.277 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-09-18T01:18:46.277 INFO:tasks.ceph:Archiving crash dumps... 2024-09-18T01:18:46.279 DEBUG:teuthology.misc:Transferring archived files from smithi028:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601/remote/smithi028/crash 2024-09-18T01:18:46.280 DEBUG:teuthology.orchestra.run.smithi028:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-09-18T01:18:46.293 DEBUG:teuthology.misc:Transferring archived files from smithi049:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601/remote/smithi049/crash 2024-09-18T01:18:46.294 DEBUG:teuthology.orchestra.run.smithi049:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-09-18T01:18:46.332 INFO:tasks.ceph:Compressing logs... 2024-09-18T01:18:46.333 DEBUG:teuthology.orchestra.run.smithi028:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-18T01:18:46.340 DEBUG:teuthology.orchestra.run.smithi049:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-18T01:18:46.362 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33205.log 2024-09-18T01:18:46.364 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28766.log 2024-09-18T01:18:46.365 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.33205.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29045.log 2024-09-18T01:18:46.365 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.33205.log.gz 2024-09-18T01:18:46.365 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28766.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28765.log 2024-09-18T01:18:46.365 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28766.log.gz 2024-09-18T01:18:46.365 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.29045.log: gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2024-09-18T01:18:46.365 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29045.log.gz 2024-09-18T01:18:46.365 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28765.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29019.log 2024-09-18T01:18:46.365 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28765.log.gz 2024-09-18T01:18:46.366 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25278.log 2024-09-18T01:18:46.366 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.29019.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28767.log 2024-09-18T01:18:46.366 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29019.log.gz 2024-09-18T01:18:46.377 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.25278.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32971.log 2024-09-18T01:18:46.377 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25278.log.gz 2024-09-18T01:18:46.378 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28767.log: gzip -5 --verbose -- 0.0% /var/log/ceph/cluster1-client.admin.33281.log -- replaced with /var/log/ceph/cluster1-client.admin.28767.log.gz 2024-09-18T01:18:46.378 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-09-18T01:18:46.378 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.32971.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2024-09-18T01:18:46.378 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.32971.log.gz 2024-09-18T01:18:46.378 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28611.log 2024-09-18T01:18:46.378 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.33281.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.33281.log.gz 2024-09-18T01:18:46.378 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25302.log 2024-09-18T01:18:46.378 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28611.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33101.log 2024-09-18T01:18:46.378 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28611.log.gz 2024-09-18T01:18:46.378 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28845.log 2024-09-18T01:18:46.378 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.25302.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25302.log.gz 2024-09-18T01:18:46.378 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.33101.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.33101.log.gz 2024-09-18T01:18:46.379 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28922.log 2024-09-18T01:18:46.379 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28845.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33049.log 2024-09-18T01:18:46.379 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28845.log.gz 2024-09-18T01:18:46.379 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2024-09-18T01:18:46.379 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28922.log: /var/log/ceph/cluster1-client.admin.33049.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28922.log.gz 2024-09-18T01:18:46.379 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.33049.log.gz 2024-09-18T01:18:46.379 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28431.log 2024-09-18T01:18:46.379 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33257.log 2024-09-18T01:18:46.379 INFO:teuthology.orchestra.run.smithi028.stderr:gzip/var/log/ceph/cluster1-client.admin.28431.log: -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.21574.log 2024-09-18T01:18:46.379 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28431.log.gz 2024-09-18T01:18:46.379 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.33257.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33127.log 2024-09-18T01:18:46.380 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.33257.log.gz 2024-09-18T01:18:46.380 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/ceph.tmp-client.admin.21574.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2024-09-18T01:18:46.380 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.21574.log.gz 2024-09-18T01:18:46.380 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2024-09-18T01:18:46.380 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.33127.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.33127.log.gz 2024-09-18T01:18:46.380 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28733.log 2024-09-18T01:18:46.380 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28837.log 2024-09-18T01:18:46.380 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2024-09-18T01:18:46.380 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28733.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28733.log.gz 2024-09-18T01:18:46.380 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33231.log 2024-09-18T01:18:46.380 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28837.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28837.log.gz 2024-09-18T01:18:46.380 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32997.log 2024-09-18T01:18:46.381 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.33231.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28915.log 2024-09-18T01:18:46.381 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.33231.log.gz 2024-09-18T01:18:46.381 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25429.log 2024-09-18T01:18:46.381 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.32997.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.32997.log.gz 2024-09-18T01:18:46.381 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25184.log 2024-09-18T01:18:46.381 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28915.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28915.log.gz 2024-09-18T01:18:46.381 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.25429.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28681.log 2024-09-18T01:18:46.381 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25429.log.gz 2024-09-18T01:18:46.381 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.25184.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25184.log.gz 2024-09-18T01:18:46.381 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33179.log 2024-09-18T01:18:46.382 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28844.log 2024-09-18T01:18:46.382 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28681.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28681.log.gz 2024-09-18T01:18:46.382 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.33179.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28661.log 2024-09-18T01:18:46.382 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.33179.log.gz 2024-09-18T01:18:46.382 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33023.log 2024-09-18T01:18:46.382 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28844.log.gz 2024-09-18T01:18:46.382 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28923.log 2024-09-18T01:18:46.382 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28661.log: /var/log/ceph/cluster1-client.admin.33023.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33075.log 2024-09-18T01:18:46.382 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.33023.log.gz 2024-09-18T01:18:46.382 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28923.log: gzip -5 --verbose -- /var/log/ceph/cluster1.log 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28923.log.gz 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.admin.28661.log.gz 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28636.log 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.33075.log: /var/log/ceph/cluster1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28707.log 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.33075.log.gz 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28636.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28636.log.gz 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25361.log 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28707.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33153.log 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28707.log.gz 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.25361.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28993.log 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25361.log.gz 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.33153.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 0.0% 2024-09-18T01:18:46.383 INFO:teuthology.orchestra.run.smithi028.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.33153.log.gz 2024-09-18T01:18:46.384 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-client.admin.28993.log: gzip 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28993.log.gz -5 2024-09-18T01:18:46.384 INFO:teuthology.orchestra.run.smithi028.stderr: --verbose -- /var/log/ceph/cluster1-client.admin.25326.log 2024-09-18T01:18:46.384 INFO:teuthology.orchestra.run.smithi028.stderr:/var/log/ceph/cluster1-osd.1.log: /var/log/ceph/cluster1-client.admin.25326.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25326.log.gz 2024-09-18T01:18:46.384 INFO:teuthology.orchestra.run.smithi028.stderr: 87.5% -- replaced with /var/log/ceph/cluster1.log.gz 2024-09-18T01:18:46.398 INFO:teuthology.orchestra.run.smithi028.stderr: 93.6% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2024-09-18T01:18:46.414 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47779.log 2024-09-18T01:18:46.415 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39054.log 2024-09-18T01:18:46.416 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47779.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80827.log 2024-09-18T01:18:46.416 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.39054.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44055.log 2024-09-18T01:18:46.418 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101984.log 2024-09-18T01:18:46.418 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80827.log: /var/log/ceph/cluster1-client.mirror.44055.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122453.log 2024-09-18T01:18:46.418 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101984.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41651.log 2024-09-18T01:18:46.418 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122453.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107174.log 2024-09-18T01:18:46.419 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41651.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121003.log 2024-09-18T01:18:46.419 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40946.log 2024-09-18T01:18:46.419 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107174.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106662.log 2024-09-18T01:18:46.419 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121003.log: /var/log/ceph/cluster2-client.mirror.40946.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130332.log 2024-09-18T01:18:46.423 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141380.log 2024-09-18T01:18:46.423 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.106662.log: /var/log/ceph/cluster1-client.mirror.130332.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34110.log 2024-09-18T01:18:46.423 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141380.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128644.log 2024-09-18T01:18:46.423 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34110.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55144.log 2024-09-18T01:18:46.423 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.128644.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97673.log 2024-09-18T01:18:46.423 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93756.log 2024-09-18T01:18:46.423 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55144.log: /var/log/ceph/cluster1-client.admin.97673.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96634.log 2024-09-18T01:18:46.424 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34110.log.gz 2024-09-18T01:18:46.424 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.93756.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116177.log 2024-09-18T01:18:46.424 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.96634.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57098.log 2024-09-18T01:18:46.424 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116177.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56344.log 2024-09-18T01:18:46.424 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.116177.log.gz 2024-09-18T01:18:46.424 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57098.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65413.log 2024-09-18T01:18:46.433 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56344.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105991.log 2024-09-18T01:18:46.434 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65413.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117948.log 2024-09-18T01:18:46.434 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.105991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99285.log 2024-09-18T01:18:46.434 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.117948.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61227.log 2024-09-18T01:18:46.434 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99285.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52266.log 2024-09-18T01:18:46.434 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.61227.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117626.log 2024-09-18T01:18:46.434 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52266.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130996.log 2024-09-18T01:18:46.434 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.117626.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127221.log 2024-09-18T01:18:46.434 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130996.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117969.log 2024-09-18T01:18:46.434 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.141380.log.gz 2024-09-18T01:18:46.435 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127221.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88495.log 2024-09-18T01:18:46.435 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117969.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59372.log 2024-09-18T01:18:46.435 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.117969.log.gz 2024-09-18T01:18:46.435 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88495.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70842.log 2024-09-18T01:18:46.435 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59372.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56974.log 2024-09-18T01:18:46.435 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70842.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34989.log 2024-09-18T01:18:46.435 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56974.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29321.log 2024-09-18T01:18:46.435 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34989.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43843.log 2024-09-18T01:18:46.435 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.29321.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84318.log 2024-09-18T01:18:46.435 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29321.log.gz 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43843.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75897.log 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84318.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92181.log 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75897.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136950.log 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92181.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58615.log 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136950.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85344.log 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58615.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59963.log 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.85344.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38395.log 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48557.log 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59963.log: /var/log/ceph/cluster1-client.admin.38395.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138292.log 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61227.log.gz 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48557.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108416.log 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138292.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95422.log 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.108416.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128580.log 2024-09-18T01:18:46.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95422.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73775.log 2024-09-18T01:18:46.437 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.128580.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114589.log 2024-09-18T01:18:46.437 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.mirror.73775.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58364.log 2024-09-18T01:18:46.444 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114589.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109582.log 2024-09-18T01:18:46.444 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58364.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109317.log 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109582.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40759.log 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65413.log.gz 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109317.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97251.log 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.40759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47991.log 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.97251.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31317.log 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109274.log 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.31317.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59205.log 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.109274.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121132.log 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59205.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104176.log 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121132.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83717.log 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140140.log 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.83717.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33165.log 2024-09-18T01:18:46.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140140.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137411.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33165.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92034.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137411.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59372.log.gz 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33584.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92034.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119734.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33584.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32578.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64245.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32578.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93232.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64245.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139071.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93232.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62469.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139071.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114800.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55144.log.gz 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.62469.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124763.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114800.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112860.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124763.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37842.log 2024-09-18T01:18:46.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112860.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130560.log 2024-09-18T01:18:46.447 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37842.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86462.log 2024-09-18T01:18:46.447 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.130560.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110890.log 2024-09-18T01:18:46.447 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86462.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33585.log 2024-09-18T01:18:46.447 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110890.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72257.log 2024-09-18T01:18:46.447 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33585.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139005.log 2024-09-18T01:18:46.447 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72257.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46893.log 2024-09-18T01:18:46.447 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139005.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133279.log 2024-09-18T01:18:46.454 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46893.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133911.log 2024-09-18T01:18:46.454 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70842.log.gz 2024-09-18T01:18:46.454 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133279.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32537.log 2024-09-18T01:18:46.454 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133911.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79945.log 2024-09-18T01:18:46.454 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32537.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53904.log 2024-09-18T01:18:46.454 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79945.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35729.log 2024-09-18T01:18:46.454 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.53904.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41779.log 2024-09-18T01:18:46.454 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35729.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37053.log 2024-09-18T01:18:46.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41779.log: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.75897.log.gz 2024-09-18T01:18:46.455 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48864.log 2024-09-18T01:18:46.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37053.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51361.log 2024-09-18T01:18:46.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48864.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113578.log 2024-09-18T01:18:46.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.51361.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81328.log 2024-09-18T01:18:46.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113578.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111892.log 2024-09-18T01:18:46.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81328.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77456.log 2024-09-18T01:18:46.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111892.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68949.log 2024-09-18T01:18:46.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77456.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49595.log 2024-09-18T01:18:46.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68949.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141246.log 2024-09-18T01:18:46.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.49595.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128753.log 2024-09-18T01:18:46.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141246.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133998.log 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.128753.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.141246.log.gz 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59081.log 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.105991.log.gz 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87270.log 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133998.log: /var/log/ceph/cluster1-client.mirror.59081.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105708.log 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.87270.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119564.log 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.105708.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99756.log 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119564.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40672.log 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99756.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117971.log 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.40672.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73116.log 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117971.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31396.log 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50645.log 2024-09-18T01:18:46.456 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31396.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42770.log 2024-09-18T01:18:46.457 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.50645.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98213.log 2024-09-18T01:18:46.457 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42770.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58509.log 2024-09-18T01:18:46.469 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98213.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43951.log 2024-09-18T01:18:46.469 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58509.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115539.log 2024-09-18T01:18:46.469 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43951.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129353.log 2024-09-18T01:18:46.469 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115539.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45587.log 2024-09-18T01:18:46.469 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129353.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117295.log 2024-09-18T01:18:46.469 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40511.log 2024-09-18T01:18:46.469 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45587.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105445.log 2024-09-18T01:18:46.469 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.117295.log: /var/log/ceph/cluster1-client.mirror.40511.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55878.log 2024-09-18T01:18:46.469 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47779.log.gz 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32955.log 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105445.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.105445.log.gz 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55878.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85230.log 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32955.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131623.log 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.85230.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36782.log 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103545.log 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131623.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123102.log 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36782.log: /var/log/ceph/cluster1-client.mirror.103545.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136570.log 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123102.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81036.log 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136570.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78253.log 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81036.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31685.log 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78253.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101100.log 2024-09-18T01:18:46.470 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.mirror.31685.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34990.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101100.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115896.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.52266.log.gz 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34990.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101228.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115896.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.132428.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101228.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107887.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.132428.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109620.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.132428.log.gz 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.107887.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65760.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109620.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85531.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65760.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91950.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.85531.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71056.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91950.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116173.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.71056.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74231.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116173.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35579.log 2024-09-18T01:18:46.471 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.116173.log.gz 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74231.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101018.log 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70606.log 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35579.log: /var/log/ceph/cluster1-client.mirror.101018.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34257.log 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70606.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40054.log 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34257.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134881.log 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.40054.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41588.log 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.40054.log.gz 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134881.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68992.log 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41588.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67891.log 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68992.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75286.log 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67891.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140614.log 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75286.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106280.log 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.140614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83606.log 2024-09-18T01:18:46.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.106280.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54031.log 2024-09-18T01:18:46.473 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83606.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84355.log 2024-09-18T01:18:46.473 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54031.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81792.log 2024-09-18T01:18:46.473 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84355.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140538.log 2024-09-18T01:18:46.473 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.107174.log.gz 2024-09-18T01:18:46.473 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97526.log 2024-09-18T01:18:46.473 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81792.log: /var/log/ceph/cluster2-client.admin.140538.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82085.log 2024-09-18T01:18:46.491 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.97526.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118611.log 2024-09-18T01:18:46.491 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82085.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120114.log 2024-09-18T01:18:46.491 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.118611.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85733.log 2024-09-18T01:18:46.491 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120114.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53555.log 2024-09-18T01:18:46.491 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85733.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126569.log 2024-09-18T01:18:46.491 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40584.log 2024-09-18T01:18:46.491 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126569.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75817.log 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.40584.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.109317.log.gz 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60329.log 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75817.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92119.log 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97163.log 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92119.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129659.log 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.97163.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32003.log 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129659.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39070.log 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129659.log.gz 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32003.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37303.log 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39070.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84783.log 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140120.log 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37303.log: /var/log/ceph/cluster2-client.admin.84783.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98575.log 2024-09-18T01:18:46.492 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84783.log.gz 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99285.log.gz 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.140120.log: /var/log/ceph/cluster1-client.admin.53201.log 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98575.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108162.log 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.53201.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122534.log 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.108162.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49092.log 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122534.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60247.log 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.49092.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98975.log 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.60247.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99369.log 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98975.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68439.log 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99369.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84787.log 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68439.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47061.log 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48557.log.gz 2024-09-18T01:18:46.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84787.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40261.log 2024-09-18T01:18:46.494 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84787.log.gz 2024-09-18T01:18:46.494 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47061.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138555.log 2024-09-18T01:18:46.494 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.40261.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93917.log 2024-09-18T01:18:46.494 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.138555.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65219.log 2024-09-18T01:18:46.494 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.138555.log.gz 2024-09-18T01:18:46.494 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93917.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37411.log 2024-09-18T01:18:46.494 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.65219.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127178.log 2024-09-18T01:18:46.494 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37411.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126715.log 2024-09-18T01:18:46.494 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57098.log.gz 2024-09-18T01:18:46.494 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61274.log 2024-09-18T01:18:46.494 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127178.log: /var/log/ceph/cluster1-client.mirror.126715.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42287.log 2024-09-18T01:18:46.494 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61274.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96540.log 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.61274.log.gz 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42287.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125185.log 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.96540.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123503.log 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89026.log 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125185.log: /var/log/ceph/cluster1-client.mirror.123503.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140531.log 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89026.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117706.log 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140531.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66190.log 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117706.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82191.log 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66190.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72407.log 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82191.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31489.log 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72407.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47739.log 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31489.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75017.log 2024-09-18T01:18:46.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47739.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99628.log 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.75017.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45630.log 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99628.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32536.log 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45630.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65003.log 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32536.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77048.log 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.65003.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107310.log 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.77048.log: --verbose -- /var/log/ceph/cluster2-client.mirror.96707.log 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107310.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108021.log 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107310.log.gz 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.96707.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103882.log 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.108021.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49540.log 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103882.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132602.log 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.49540.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106858.log 2024-09-18T01:18:46.496 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98361.log 2024-09-18T01:18:46.497 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132602.log: /var/log/ceph/cluster2-client.admin.106858.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47105.log 2024-09-18T01:18:46.497 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.98361.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28447.log 2024-09-18T01:18:46.497 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47105.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97579.log 2024-09-18T01:18:46.497 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28447.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102108.log 2024-09-18T01:18:46.497 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28447.log.gz 2024-09-18T01:18:46.497 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.97579.log: --verbose -- /var/log/ceph/cluster1-client.mirror.115600.log 2024-09-18T01:18:46.497 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102108.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36003.log 2024-09-18T01:18:46.497 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115600.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72119.log 2024-09-18T01:18:46.497 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36003.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120324.log 2024-09-18T01:18:46.497 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72119.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32004.log 2024-09-18T01:18:46.497 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120324.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60877.log 2024-09-18T01:18:46.498 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.mirror.32004.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111110.log 2024-09-18T01:18:46.498 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130332.log.gz 2024-09-18T01:18:46.498 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.60877.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74392.log 2024-09-18T01:18:46.498 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111110.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75849.log 2024-09-18T01:18:46.498 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.74392.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82296.log 2024-09-18T01:18:46.498 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.75849.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113961.log 2024-09-18T01:18:46.551 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82296.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60108.log 2024-09-18T01:18:46.551 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113961.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112525.log 2024-09-18T01:18:46.551 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60108.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86105.log 2024-09-18T01:18:46.551 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60108.log.gz 2024-09-18T01:18:46.551 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112525.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138128.log 2024-09-18T01:18:46.551 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.86105.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103796.log 2024-09-18T01:18:46.551 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.86105.log.gz 2024-09-18T01:18:46.552 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138128.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60160.log 2024-09-18T01:18:46.552 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103796.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111748.log 2024-09-18T01:18:46.552 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.60160.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91405.log 2024-09-18T01:18:46.552 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103964.log 2024-09-18T01:18:46.552 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111748.log: /var/log/ceph/cluster1-client.mirror.91405.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108416.log.gz 2024-09-18T01:18:46.552 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101814.log 2024-09-18T01:18:46.552 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103964.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74532.log 2024-09-18T01:18:46.552 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101814.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110146.log 2024-09-18T01:18:46.552 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.74532.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59582.log 2024-09-18T01:18:46.552 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110146.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125391.log 2024-09-18T01:18:46.552 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59582.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36295.log 2024-09-18T01:18:46.552 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104616.log 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125391.log: /var/log/ceph/cluster1-client.mirror.36295.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142442.log 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.109582.log.gz 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72986.log 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.142442.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142442.log.gz 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44411.log 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster2-client.mirror.72986.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104988.log 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44411.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62318.log 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104988.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112522.log 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.62318.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133868.log 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62318.log.gz 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112522.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39565.log 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133868.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106661.log 2024-09-18T01:18:46.553 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39565.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67389.log 2024-09-18T01:18:46.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.106661.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106948.log 2024-09-18T01:18:46.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67389.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49959.log 2024-09-18T01:18:46.554 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75102.log 2024-09-18T01:18:46.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106948.log: /var/log/ceph/cluster1-client.admin.49959.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113534.log 2024-09-18T01:18:46.554 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81792.log.gz 2024-09-18T01:18:46.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75102.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81457.log 2024-09-18T01:18:46.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113534.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107390.log 2024-09-18T01:18:46.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81457.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130466.log 2024-09-18T01:18:46.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.107390.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138558.log 2024-09-18T01:18:46.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130466.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132106.log 2024-09-18T01:18:46.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138558.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140328.log 2024-09-18T01:18:46.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132106.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129481.log 2024-09-18T01:18:46.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140328.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138358.log 2024-09-18T01:18:46.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129481.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130876.log 2024-09-18T01:18:46.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138358.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100175.log 2024-09-18T01:18:46.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130876.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131558.log 2024-09-18T01:18:46.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100175.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46093.log 2024-09-18T01:18:46.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131558.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57139.log 2024-09-18T01:18:46.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46093.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126000.log 2024-09-18T01:18:46.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57139.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36151.log 2024-09-18T01:18:46.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126000.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138005.log 2024-09-18T01:18:46.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36151.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53243.log 2024-09-18T01:18:46.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138005.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112378.log 2024-09-18T01:18:46.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53243.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118839.log 2024-09-18T01:18:46.556 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112378.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91553.log 2024-09-18T01:18:46.556 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.118839.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60767.log 2024-09-18T01:18:46.556 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91553.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38926.log 2024-09-18T01:18:46.556 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.60767.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132469.log 2024-09-18T01:18:46.556 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.38926.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48411.log 2024-09-18T01:18:46.556 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108718.log 2024-09-18T01:18:46.556 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132469.log: /var/log/ceph/cluster1-client.mirror.48411.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29857.log 2024-09-18T01:18:46.556 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39373.log 2024-09-18T01:18:46.556 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.108718.log: /var/log/ceph/cluster2-client.mirror.29857.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45671.log 2024-09-18T01:18:46.556 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39373.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80995.log 2024-09-18T01:18:46.556 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.39373.log.gz 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45671.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91973.log 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80995.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41454.log 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84806.log 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91973.log: /var/log/ceph/cluster1-client.mirror.41454.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134840.log 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34989.log.gz 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34990.log.gz 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84806.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57818.log 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56344.log.gz 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134840.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128508.log 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57818.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37577.log 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.128508.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136990.log 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37577.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139758.log 2024-09-18T01:18:46.557 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136990.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125579.log 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139758.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49006.log 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125579.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99389.log 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.49006.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128103.log 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99389.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131491.log 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.128103.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131468.log 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.131491.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135134.log 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86464.log 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131468.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.58364.log.gz 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135134.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65759.log 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86464.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66841.log 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65759.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38800.log 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66841.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131760.log 2024-09-18T01:18:46.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.38800.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113364.log 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131760.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29942.log 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113364.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74593.log 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.29942.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115326.log 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.74593.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52944.log 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75561.log 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115326.log: /var/log/ceph/cluster1-client.admin.52944.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105855.log 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70713.log 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75561.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.88495.log.gz 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.105855.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28664.log 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70713.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128101.log 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28664.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45564.log 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28664.log.gz 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.128101.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79646.log 2024-09-18T01:18:46.559 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45564.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39994.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79646.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129949.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39994.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73372.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.129949.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57055.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73372.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105640.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57055.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40812.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104512.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.105640.log: /var/log/ceph/cluster2-client.mirror.40812.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57899.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104512.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80028.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57899.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40147.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80028.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109263.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.40147.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90917.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.109263.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95238.log 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.109263.log.gz 2024-09-18T01:18:46.560 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50250.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90917.log: /var/log/ceph/cluster2-client.admin.95238.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49136.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.50250.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136610.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64280.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.49136.log: /var/log/ceph/cluster1-client.mirror.136610.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114487.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64280.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87003.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114487.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65497.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73502.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87003.log: /var/log/ceph/cluster1-client.mirror.65497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65347.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.73502.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35032.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65347.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78126.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35032.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39936.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80827.log.gz 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79986.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39936.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128586.log 2024-09-18T01:18:46.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79986.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50847.log 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87670.log 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.128586.log: /var/log/ceph/cluster2-client.mirror.50847.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106744.log 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87670.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76316.log 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106744.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83396.log 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48199.log 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76316.log: /var/log/ceph/cluster2-client.admin.83396.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56638.log 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48199.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61228.log 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47778.log 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56638.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109696.log 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.61228.log: /var/log/ceph/cluster1-client.mirror.47778.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122243.log 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.109696.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140689.log 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59205.log.gz 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.109696.log.gz 2024-09-18T01:18:46.562 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122243.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110781.log 2024-09-18T01:18:46.563 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140689.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84900.log 2024-09-18T01:18:46.563 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110781.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109177.log 2024-09-18T01:18:46.563 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37515.log 2024-09-18T01:18:46.563 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84900.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84900.log.gz 2024-09-18T01:18:46.563 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109177.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35264.log 2024-09-18T01:18:46.563 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37515.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82910.log 2024-09-18T01:18:46.563 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35264.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.140851.log 2024-09-18T01:18:46.563 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82910.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102998.log 2024-09-18T01:18:46.563 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.140851.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36042.log 2024-09-18T01:18:46.563 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102998.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123294.log 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.140851.log.gz 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127054.log 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36042.log: /var/log/ceph/cluster1-client.mirror.123294.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51911.log 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127054.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59246.log 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.51911.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62925.log 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59246.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53369.log 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.62925.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71359.log 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.53369.log: gzip 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.53369.log.gz -5 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.133658.log 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.71359.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109734.log 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133658.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122665.log 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68523.log 2024-09-18T01:18:46.564 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122665.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70418.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68523.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100428.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70418.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41805.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100428.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60481.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41805.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40645.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94516.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.60481.log: /var/log/ceph/cluster1-client.admin.40645.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106818.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88604.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94516.log: /var/log/ceph/cluster2-client.admin.106818.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112692.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88604.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97742.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112692.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28509.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97742.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.93857.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28509.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105978.log 2024-09-18T01:18:46.565 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28509.log.gz 2024-09-18T01:18:46.566 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.93857.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39064.log 2024-09-18T01:18:46.566 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.105978.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121467.log 2024-09-18T01:18:46.566 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.39064.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69158.log 2024-09-18T01:18:46.566 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85049.log 2024-09-18T01:18:46.566 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121467.log: /var/log/ceph/cluster1-client.mirror.69158.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123589.log 2024-09-18T01:18:46.566 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85049.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80700.log 2024-09-18T01:18:46.566 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123589.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42473.log 2024-09-18T01:18:46.566 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80700.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100763.log 2024-09-18T01:18:46.566 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.42473.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43192.log 2024-09-18T01:18:46.566 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117922.log 2024-09-18T01:18:46.566 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100763.log: /var/log/ceph/cluster1-client.mirror.43192.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124824.log 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.96634.log.gz 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117922.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77369.log 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124824.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46219.log 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77369.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97933.log 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46219.log: gzip -5 --verbose 66.8% -- /var/log/ceph/cluster1-client.mirror.123926.log -- replaced with /var/log/ceph/cluster1-client.mirror.85344.log.gz 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.97933.log: -- /var/log/ceph/cluster1-client.mirror.80531.log 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123926.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48325.log 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86158.log 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80531.log: /var/log/ceph/cluster1-client.mirror.48325.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115389.log 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102702.log 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86158.log: /var/log/ceph/cluster1-client.mirror.115389.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44371.log 2024-09-18T01:18:46.567 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.102702.log: --verbose -- /var/log/ceph/cluster1-client.mirror.47738.log 2024-09-18T01:18:46.568 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44371.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52671.log 2024-09-18T01:18:46.568 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47738.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137371.log 2024-09-18T01:18:46.568 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52671.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51133.log 2024-09-18T01:18:46.568 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137371.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94559.log 2024-09-18T01:18:46.568 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95423.log 2024-09-18T01:18:46.568 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.51133.log: /var/log/ceph/cluster1-client.admin.94559.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45102.log 2024-09-18T01:18:46.568 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95423.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70672.log 2024-09-18T01:18:46.568 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45102.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127580.log 2024-09-18T01:18:46.568 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124904.log 2024-09-18T01:18:46.568 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70672.log: /var/log/ceph/cluster1-client.mirror.127580.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136168.log 2024-09-18T01:18:46.568 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.124904.log: --verbose -- /var/log/ceph/cluster1-client.mirror.56763.log 2024-09-18T01:18:46.568 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136168.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42464.log 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.97251.log.gz 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71323.log 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56763.log: /var/log/ceph/cluster1-client.admin.42464.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110269.log 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.71323.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126866.log 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110269.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52216.log 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133196.log 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126866.log: /var/log/ceph/cluster1-client.admin.52216.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65804.log 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.52216.log.gz 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133196.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98246.log 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93796.log 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65804.log: /var/log/ceph/cluster1-client.mirror.98246.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50985.log 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.admin.93796.log: -- /var/log/ceph/cluster2-client.admin.139753.log 2024-09-18T01:18:46.569 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.93796.log.gz 2024-09-18T01:18:46.570 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50985.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108585.log 2024-09-18T01:18:46.570 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50985.log.gz 2024-09-18T01:18:46.570 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59499.log 2024-09-18T01:18:46.570 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139753.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.139753.log.gz 2024-09-18T01:18:46.570 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.108585.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120998.log 2024-09-18T01:18:46.570 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.59499.log: --verbose -- /var/log/ceph/cluster1-client.mirror.47145.log 2024-09-18T01:18:46.570 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120998.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134248.log 2024-09-18T01:18:46.570 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124890.log 2024-09-18T01:18:46.570 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47145.log: /var/log/ceph/cluster1-client.mirror.134248.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35877.log 2024-09-18T01:18:46.687 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38801.log 2024-09-18T01:18:46.687 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124890.log: /var/log/ceph/cluster1-client.mirror.35877.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113322.log 2024-09-18T01:18:46.687 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster2-client.admin.38801.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110335.log 2024-09-18T01:18:46.687 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113322.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.121131.log 2024-09-18T01:18:46.687 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110335.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82738.log 2024-09-18T01:18:46.688 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.121131.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99219.log 2024-09-18T01:18:46.688 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.121131.log.gz 2024-09-18T01:18:46.688 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.admin.82738.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48052.log 2024-09-18T01:18:46.688 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99219.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79734.log 2024-09-18T01:18:46.688 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48052.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30821.log 2024-09-18T01:18:46.688 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45168.log 2024-09-18T01:18:46.688 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79734.log: /var/log/ceph/cluster1-client.admin.30821.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63427.log 2024-09-18T01:18:46.688 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45168.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134715.log 2024-09-18T01:18:46.688 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.63427.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107303.log 2024-09-18T01:18:46.688 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123984.log 2024-09-18T01:18:46.688 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134715.log: /var/log/ceph/cluster2-client.admin.107303.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101984.log.gz 2024-09-18T01:18:46.688 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0%gzip -- replaced with /var/log/ceph/cluster2-client.admin.107303.log.gz 2024-09-18T01:18:46.689 INFO:teuthology.orchestra.run.smithi049.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113918.log 2024-09-18T01:18:46.689 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62401.log 2024-09-18T01:18:46.689 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123984.log: /var/log/ceph/cluster1-client.mirror.113918.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77004.log 2024-09-18T01:18:46.689 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88496.log 2024-09-18T01:18:46.689 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.62401.log: /var/log/ceph/cluster1-client.mirror.77004.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127842.log 2024-09-18T01:18:46.689 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88496.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.132214.log 2024-09-18T01:18:46.689 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.127842.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95738.log 2024-09-18T01:18:46.689 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.132214.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131346.log 2024-09-18T01:18:46.689 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95738.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59080.log 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115266.log 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.131346.log: /var/log/ceph/cluster1-client.mirror.59080.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104384.log 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115266.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45523.log 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104384.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75586.log 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133238.log 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45523.log: /var/log/ceph/cluster1-client.mirror.75586.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35792.log 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133238.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46512.log 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35792.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59204.log 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44370.log 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46512.log: /var/log/ceph/cluster1-client.mirror.59204.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57690.log 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44370.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104177.log 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.117626.log.gz 2024-09-18T01:18:46.690 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57690.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65669.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104177.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80406.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65669.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76225.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80406.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105226.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.76225.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110546.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.105226.log: -- /var/log/ceph/cluster1-client.admin.49339.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110546.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89715.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.49339.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127581.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.49339.log.gz 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89715.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91529.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41672.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127581.log: /var/log/ceph/cluster1-client.mirror.91529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112314.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86683.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41672.log: /var/log/ceph/cluster1-client.mirror.112314.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39067.log 2024-09-18T01:18:46.691 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121702.log 2024-09-18T01:18:46.692 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86683.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.106662.log.gz 2024-09-18T01:18:46.692 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.39067.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.29998.log 2024-09-18T01:18:46.692 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121702.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120288.log 2024-09-18T01:18:46.692 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.0.29998.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89025.log 2024-09-18T01:18:46.692 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120288.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84904.log 2024-09-18T01:18:46.692 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89025.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116933.log 2024-09-18T01:18:46.692 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster2-client.admin.84904.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103039.log 2024-09-18T01:18:46.692 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84904.log.gz 2024-09-18T01:18:46.692 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116933.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60501.log 2024-09-18T01:18:46.692 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103039.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95679.log 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60501.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78419.log 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91339.log 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95679.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.95679.log.gz 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78419.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89863.log 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91339.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60512.log 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89863.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114002.log 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86289.log 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60512.log: /var/log/ceph/cluster1-client.mirror.114002.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68993.log 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86289.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68400.log 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68993.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91129.log 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68400.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79690.log 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40698.log 2024-09-18T01:18:46.693 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91129.log: /var/log/ceph/cluster1-client.mirror.79690.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87853.log 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121787.log 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9%/var/log/ceph/cluster2-client.mirror.40698.log: -- replaced with /var/log/ceph/cluster1-client.mirror.114800.log.gz 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87853.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34573.log 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44581.log 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121787.log: /var/log/ceph/cluster1-client.mirror.34573.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130291.log 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72290.log 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44581.log: /var/log/ceph/cluster2-client.mirror.130291.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140266.log 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72290.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43780.log 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140266.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44620.log 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140266.log.gz 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101941.log 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43780.log: /var/log/ceph/cluster1-client.mirror.44620.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122872.log 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101941.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82849.log 2024-09-18T01:18:46.694 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122872.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84786.log 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82849.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76769.log 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84786.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76087.log 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76769.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102955.log 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122453.log.gz 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76087.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37907.log 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102955.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35793.log 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37907.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74164.log 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35793.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107960.log 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.74164.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49740.log 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42771.log 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.107960.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.117948.log.gz 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.49740.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141515.log 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67051.log 2024-09-18T01:18:46.695 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42771.log: /var/log/ceph/cluster1-client.mirror.141515.log: 67.8% -- replaced with /var/log/ceph/cluster1-client.mirror.141515.log.gz 2024-09-18T01:18:46.696 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100974.log 2024-09-18T01:18:46.696 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.mirror.67051.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44682.log 2024-09-18T01:18:46.696 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100974.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61763.log 2024-09-18T01:18:46.696 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44682.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134923.log 2024-09-18T01:18:46.696 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61763.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139425.log 2024-09-18T01:18:46.696 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134923.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139426.log 2024-09-18T01:18:46.696 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139425.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97664.log 2024-09-18T01:18:46.696 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.139425.log.gz 2024-09-18T01:18:46.696 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72742.log 2024-09-18T01:18:46.696 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139426.log: /var/log/ceph/cluster1-client.admin.97664.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82931.log 2024-09-18T01:18:46.696 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.72742.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54463.log 2024-09-18T01:18:46.696 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.admin.82931.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.105489.log 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.82931.log.gz 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54463.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134418.log 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105489.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122142.log 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134418.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61979.log 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122142.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85189.log 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133998.log.gz 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61979.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73734.log 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.85189.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129096.log 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.73734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74076.log 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.129096.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114213.log 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.74076.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129011.log 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114213.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136612.log 2024-09-18T01:18:46.697 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.129011.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132558.log 2024-09-18T01:18:46.698 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136612.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64754.log 2024-09-18T01:18:46.698 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132558.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97406.log 2024-09-18T01:18:46.698 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.64754.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111432.log 2024-09-18T01:18:46.698 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.97406.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125161.log 2024-09-18T01:18:46.698 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111432.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139777.log 2024-09-18T01:18:46.698 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125161.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108074.log 2024-09-18T01:18:46.698 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139777.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73661.log 2024-09-18T01:18:46.698 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137411.log.gz 2024-09-18T01:18:46.699 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.108074.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134717.log 2024-09-18T01:18:46.699 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73661.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51341.log 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73661.log.gz 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134717.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82297.log 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.51341.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60389.log 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82297.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79735.log 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113011.log 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60389.log: /var/log/ceph/cluster1-client.mirror.79735.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66525.log 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136950.log.gz 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113011.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85250.log 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66525.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38549.log 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85250.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84271.log 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.38549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39660.log 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84271.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127527.log 2024-09-18T01:18:46.700 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.39660.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112902.log 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster1-client.admin.130996.log.gz 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.127527.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137872.log 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.127527.log.gz 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120546.log 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112902.log: /var/log/ceph/cluster1-client.mirror.137872.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62784.log 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120546.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138003.log 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.120546.log.gz 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.62784.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29686.log 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138003.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92875.log 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.29686.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94515.log 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92875.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63929.log 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94515.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120863.log 2024-09-18T01:18:46.701 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.63929.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139309.log 2024-09-18T01:18:46.702 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.63929.log.gz 2024-09-18T01:18:46.702 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120863.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48200.log 2024-09-18T01:18:46.702 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.139309.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49668.log 2024-09-18T01:18:46.702 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48200.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130747.log 2024-09-18T01:18:46.702 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.49668.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86425.log 2024-09-18T01:18:46.702 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.130747.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56299.log 2024-09-18T01:18:46.702 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.86425.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122015.log 2024-09-18T01:18:46.702 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56299.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117471.log 2024-09-18T01:18:46.702 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122015.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50517.log 2024-09-18T01:18:46.703 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117471.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51662.log 2024-09-18T01:18:46.703 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50517.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51528.log 2024-09-18T01:18:46.703 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51662.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129653.log 2024-09-18T01:18:46.703 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51528.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98350.log 2024-09-18T01:18:46.703 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73553.log 2024-09-18T01:18:46.703 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129653.log.gz 2024-09-18T01:18:46.703 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.98350.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67936.log 2024-09-18T01:18:46.703 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.98350.log.gz 2024-09-18T01:18:46.703 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73553.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60812.log 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73553.log.gz 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67936.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88102.log 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.60812.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119894.log 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88102.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84034.log 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.119894.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125811.log 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.119894.log.gz 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.128644.log.gz 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.84034.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120752.log 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125811.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81161.log 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120752.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.129809.log 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53981.log 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81161.log: /var/log/ceph/cluster1-client.admin.129809.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62042.log 2024-09-18T01:18:46.704 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53981.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137456.log 2024-09-18T01:18:46.705 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.62042.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135987.log 2024-09-18T01:18:46.705 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137456.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102150.log 2024-09-18T01:18:46.705 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135987.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128835.log 2024-09-18T01:18:46.705 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102150.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35835.log 2024-09-18T01:18:46.705 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.128835.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91066.log 2024-09-18T01:18:46.705 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35835.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139959.log 2024-09-18T01:18:46.705 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91066.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58196.log 2024-09-18T01:18:46.705 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64936.log 2024-09-18T01:18:46.705 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139959.log: /var/log/ceph/cluster1-client.mirror.58196.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32639.log 2024-09-18T01:18:46.705 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64936.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91971.log 2024-09-18T01:18:46.705 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32639.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40397.log 2024-09-18T01:18:46.706 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91971.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70547.log 2024-09-18T01:18:46.706 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.40397.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98282.log 2024-09-18T01:18:46.706 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70547.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125391.log.gz 2024-09-18T01:18:46.706 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83468.log 2024-09-18T01:18:46.706 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98282.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68781.log 2024-09-18T01:18:46.706 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.83468.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139653.log 2024-09-18T01:18:46.706 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68781.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57877.log 2024-09-18T01:18:46.706 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.139653.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105637.log 2024-09-18T01:18:46.706 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57877.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89298.log 2024-09-18T01:18:46.706 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.105637.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79049.log 2024-09-18T01:18:46.706 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89298.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46936.log 2024-09-18T01:18:46.706 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79049.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105221.log 2024-09-18T01:18:46.707 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46936.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108716.log 2024-09-18T01:18:46.707 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100723.log 2024-09-18T01:18:46.707 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105221.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.105221.log.gz 2024-09-18T01:18:46.707 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120761.log 2024-09-18T01:18:46.707 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.108716.log: /var/log/ceph/cluster1-client.mirror.100723.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77767.log 2024-09-18T01:18:46.707 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120761.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30985.log 2024-09-18T01:18:46.707 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77767.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97204.log 2024-09-18T01:18:46.707 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.30985.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137665.log 2024-09-18T01:18:46.707 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97204.log: 89.9%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.138292.log.gz -5 2024-09-18T01:18:46.707 INFO:teuthology.orchestra.run.smithi049.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.79900.log 2024-09-18T01:18:46.707 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137665.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.118786.log 2024-09-18T01:18:46.708 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79900.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118246.log 2024-09-18T01:18:46.708 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.118786.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77874.log 2024-09-18T01:18:46.708 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118246.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84400.log 2024-09-18T01:18:46.708 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77874.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34363.log 2024-09-18T01:18:46.708 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.84400.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33312.log 2024-09-18T01:18:46.708 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104342.log 2024-09-18T01:18:46.708 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34363.log: /var/log/ceph/cluster1-client.mirror.33312.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71119.log 2024-09-18T01:18:46.708 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104342.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113576.log 2024-09-18T01:18:46.708 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82444.log 2024-09-18T01:18:46.708 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.71119.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130994.log 2024-09-18T01:18:46.708 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113576.log: /var/log/ceph/cluster1-client.mirror.82444.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32220.log 2024-09-18T01:18:46.709 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130994.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58240.log 2024-09-18T01:18:46.709 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.130994.log.gz 2024-09-18T01:18:46.709 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.mirror.32220.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56678.log 2024-09-18T01:18:46.709 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58240.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103376.log 2024-09-18T01:18:46.709 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56678.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125099.log 2024-09-18T01:18:46.709 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103376.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67979.log 2024-09-18T01:18:46.709 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125099.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.106948.log.gz 2024-09-18T01:18:46.709 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72513.log 2024-09-18T01:18:46.709 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67979.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103925.log 2024-09-18T01:18:46.709 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.72513.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89295.log 2024-09-18T01:18:46.709 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103925.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30440.log 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89295.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70989.log 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.30440.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110991.log 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56298.log 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70989.log: /var/log/ceph/cluster1-client.mirror.110991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73795.log 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56298.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107618.log 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.73795.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41743.log 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.107618.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35370.log 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41743.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71612.log 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35370.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39484.log 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127221.log.gz 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71612.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39362.log 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39484.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48092.log 2024-09-18T01:18:46.710 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39362.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77108.log 2024-09-18T01:18:46.711 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.39362.log.gz 2024-09-18T01:18:46.711 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56553.log 2024-09-18T01:18:46.711 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48092.log: /var/log/ceph/cluster1-client.mirror.77108.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129133.log 2024-09-18T01:18:46.711 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56553.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129663.log 2024-09-18T01:18:46.711 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.129133.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33270.log 2024-09-18T01:18:46.711 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129663.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69202.log 2024-09-18T01:18:46.711 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129663.log.gz 2024-09-18T01:18:46.711 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62629.log 2024-09-18T01:18:46.711 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33270.log: /var/log/ceph/cluster1-client.mirror.69202.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122144.log 2024-09-18T01:18:46.711 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.62629.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122533.log 2024-09-18T01:18:46.711 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122144.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119336.log 2024-09-18T01:18:46.712 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122533.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32219.log 2024-09-18T01:18:46.712 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119336.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87632.log 2024-09-18T01:18:46.712 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32219.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.121467.log.gz 2024-09-18T01:18:46.712 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82612.log 2024-09-18T01:18:46.712 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87632.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59291.log 2024-09-18T01:18:46.712 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82612.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78797.log 2024-09-18T01:18:46.712 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121760.log 2024-09-18T01:18:46.712 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59291.log: /var/log/ceph/cluster1-client.mirror.78797.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132108.log 2024-09-18T01:18:46.712 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121760.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97049.log 2024-09-18T01:18:46.713 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132108.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117828.log 2024-09-18T01:18:46.713 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.97049.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34530.log 2024-09-18T01:18:46.713 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117828.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101268.log 2024-09-18T01:18:46.713 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34530.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37705.log 2024-09-18T01:18:46.713 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101268.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71963.log 2024-09-18T01:18:46.713 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37705.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82684.log 2024-09-18T01:18:46.713 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.71963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83514.log 2024-09-18T01:18:46.713 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82684.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72823.log 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.83514.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131335.log 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.72823.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119108.log 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.admin.131335.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133827.log 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.131335.log.gz 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119108.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46893.log.gz 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76216.log 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133827.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71421.log 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.76216.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63065.log 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38631.log 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71421.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.71421.log.gz 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.63065.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124554.log 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.38631.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73021.log 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124554.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57817.log 2024-09-18T01:18:46.714 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73021.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127939.log 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57817.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74096.log 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.127939.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46516.log 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.74096.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41408.log 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46219.log.gz 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52935.log 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46516.log: /var/log/ceph/cluster1-client.admin.41408.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39059.log 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.52935.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47529.log 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52175.log 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.39059.log: /var/log/ceph/cluster1-client.mirror.47529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43573.log 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52175.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78083.log 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43573.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59665.log 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56273.log 2024-09-18T01:18:46.715 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78083.log: /var/log/ceph/cluster1-client.mirror.59665.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36463.log 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56273.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53581.log 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68400.log.gz 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36463.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108534.log 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141819.log 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53581.log: /var/log/ceph/cluster1-client.admin.108534.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40062.log 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.108534.log.gz 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141819.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96151.log 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.40062.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.141819.log.gz 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67449.log 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.40062.log.gz 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42876.log 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.96151.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96151.log.gz 2024-09-18T01:18:46.716 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67449.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114635.log 2024-09-18T01:18:46.717 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42876.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93709.log 2024-09-18T01:18:46.717 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51388.log 2024-09-18T01:18:46.717 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114635.log: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.74164.log.gz 2024-09-18T01:18:46.717 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93709.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116546.log 2024-09-18T01:18:46.717 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.51388.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34889.log 2024-09-18T01:18:46.717 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116546.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.116546.log.gz 2024-09-18T01:18:46.717 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99411.log 2024-09-18T01:18:46.718 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34889.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99715.log 2024-09-18T01:18:46.718 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99411.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132149.log 2024-09-18T01:18:46.718 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99715.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33731.log 2024-09-18T01:18:46.718 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132149.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67391.log 2024-09-18T01:18:46.718 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33731.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133617.log 2024-09-18T01:18:46.718 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67391.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43572.log 2024-09-18T01:18:46.718 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117491.log 2024-09-18T01:18:46.718 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133617.log: /var/log/ceph/cluster1-client.mirror.43572.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60157.log 2024-09-18T01:18:46.718 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117491.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53664.log 2024-09-18T01:18:46.719 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60157.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103923.log 2024-09-18T01:18:46.719 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60157.log.gz 2024-09-18T01:18:46.719 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53664.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44934.log 2024-09-18T01:18:46.719 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103923.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113433.log 2024-09-18T01:18:46.719 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124890.log.gz 2024-09-18T01:18:46.719 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.124904.log.gz 2024-09-18T01:18:46.719 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137661.log 2024-09-18T01:18:46.719 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44934.log: /var/log/ceph/cluster1-client.mirror.113433.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141132.log 2024-09-18T01:18:46.719 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112796.log 2024-09-18T01:18:46.719 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137661.log: /var/log/ceph/cluster2-client.mirror.141132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130037.log 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112796.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.141132.log.gz 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51642.log 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130037.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85800.log 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51642.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85002.log 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.85800.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89002.log 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.85002.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34048.log 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.mirror.89002.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99413.log 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34048.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55354.log 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99413.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.107679.log 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55354.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91823.log 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.107679.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77915.log 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91823.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37820.log 2024-09-18T01:18:46.720 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77915.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125100.log 2024-09-18T01:18:46.721 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.37820.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103165.log 2024-09-18T01:18:46.721 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125100.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109066.log 2024-09-18T01:18:46.721 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103165.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124132.log 2024-09-18T01:18:46.721 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48577.log 2024-09-18T01:18:46.721 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109066.log: /var/log/ceph/cluster1-client.mirror.124132.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111748.log.gz 2024-09-18T01:18:46.721 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45377.log 2024-09-18T01:18:46.721 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48577.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97183.log 2024-09-18T01:18:46.721 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45377.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57731.log 2024-09-18T01:18:46.721 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.97183.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60300.log 2024-09-18T01:18:46.721 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57731.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60390.log 2024-09-18T01:18:46.721 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60300.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90137.log 2024-09-18T01:18:46.721 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60300.log.gz 2024-09-18T01:18:46.722 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60390.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38396.log 2024-09-18T01:18:46.722 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90137.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113072.log 2024-09-18T01:18:46.722 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.38396.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121397.log 2024-09-18T01:18:46.722 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102618.log 2024-09-18T01:18:46.722 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113072.log: /var/log/ceph/cluster1-client.mirror.121397.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33836.log 2024-09-18T01:18:46.722 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102618.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122077.log 2024-09-18T01:18:46.722 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33836.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97354.log 2024-09-18T01:18:46.722 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104135.log 2024-09-18T01:18:46.722 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122077.log: /var/log/ceph/cluster1-client.mirror.97354.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122579.log 2024-09-18T01:18:46.722 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104135.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69495.log 2024-09-18T01:18:46.722 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122579.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104552.log 2024-09-18T01:18:46.722 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69495.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77244.log 2024-09-18T01:18:46.723 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.31685.log.gz 2024-09-18T01:18:46.723 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104552.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45946.log 2024-09-18T01:18:46.723 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77244.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103713.log 2024-09-18T01:18:46.723 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45946.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129583.log 2024-09-18T01:18:46.723 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96186.log 2024-09-18T01:18:46.723 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103713.log: /var/log/ceph/cluster2-client.admin.129583.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64555.log 2024-09-18T01:18:46.723 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129583.log.gz 2024-09-18T01:18:46.723 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.96186.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86619.log 2024-09-18T01:18:46.723 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96186.log.gz 2024-09-18T01:18:46.724 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.64555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141702.log 2024-09-18T01:18:46.724 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86619.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78798.log 2024-09-18T01:18:46.724 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141702.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124889.log 2024-09-18T01:18:46.724 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78798.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.141702.log.gz 2024-09-18T01:18:46.724 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50174.log 2024-09-18T01:18:46.724 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124889.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66013.log 2024-09-18T01:18:46.724 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50174.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112316.log 2024-09-18T01:18:46.724 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66013.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110891.log 2024-09-18T01:18:46.724 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32639.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.32004.log.gz 2024-09-18T01:18:46.724 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:46.724 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112316.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55060.log 2024-09-18T01:18:46.725 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110891.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47146.log 2024-09-18T01:18:46.725 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55060.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62355.log 2024-09-18T01:18:46.725 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47146.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104807.log 2024-09-18T01:18:46.725 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.62355.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99456.log 2024-09-18T01:18:46.725 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104807.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126446.log 2024-09-18T01:18:46.725 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99456.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99672.log 2024-09-18T01:18:46.725 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139871.log 2024-09-18T01:18:46.725 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126446.log: /var/log/ceph/cluster1-client.mirror.99672.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123631.log 2024-09-18T01:18:46.725 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139871.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70085.log 2024-09-18T01:18:46.725 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123631.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.132028.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70085.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109088.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.132028.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32360.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.114589.log.gz 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109088.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115264.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32360.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30302.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115264.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119450.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.30302.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77413.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119450.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39045.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77413.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113745.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.39045.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86910.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113745.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37600.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.86910.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54529.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48578.log 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37600.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47991.log.gz 2024-09-18T01:18:46.726 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134288.log 2024-09-18T01:18:46.727 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48578.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91275.log 2024-09-18T01:18:46.727 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134288.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80994.log 2024-09-18T01:18:46.727 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29271.log 2024-09-18T01:18:46.727 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80994.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51274.log 2024-09-18T01:18:46.727 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.29271.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42770.log.gz 2024-09-18T01:18:46.727 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39983.log 2024-09-18T01:18:46.727 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29271.log.gz 2024-09-18T01:18:46.727 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.51274.log: /var/log/ceph/cluster1-client.mirror.91275.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119470.log 2024-09-18T01:18:46.727 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39983.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75648.log 2024-09-18T01:18:46.727 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.119470.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125392.log 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59879.log 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.75648.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.75648.log.gz 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125392.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68314.log 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59879.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38254.log 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68314.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.33165.log.gz 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70043.log 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.38254.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119896.log 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.mirror.70043.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.71290.log 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.119896.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139535.log 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71290.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65927.log 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.71290.log.gz 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39909.log 2024-09-18T01:18:46.728 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139535.log: /var/log/ceph/cluster1-client.mirror.65927.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39116.log 2024-09-18T01:18:46.729 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79603.log 2024-09-18T01:18:46.729 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86064.log 2024-09-18T01:18:46.729 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79603.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109698.log 2024-09-18T01:18:46.729 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83717.log.gz 2024-09-18T01:18:46.729 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86064.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47630.log 2024-09-18T01:18:46.729 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.109698.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92327.log 2024-09-18T01:18:46.729 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47630.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136194.log 2024-09-18T01:18:46.729 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92327.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107023.log 2024-09-18T01:18:46.729 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136194.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135980.log 2024-09-18T01:18:46.729 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107023.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84559.log 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135980.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66318.log 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84559.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78588.log 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35579.log.gz 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39692.log 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66318.log: /var/log/ceph/cluster1-client.mirror.78588.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50147.log 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39692.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37201.log 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50147.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54974.log 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37201.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141629.log 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54974.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130446.log 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.141629.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92453.log 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr: 67.5% -- replaced with /var/log/ceph/cluster1-client.mirror.141629.log.gz 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.130446.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124972.log 2024-09-18T01:18:46.730 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92453.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141173.log 2024-09-18T01:18:46.731 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124972.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96613.log 2024-09-18T01:18:46.731 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.141173.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73681.log 2024-09-18T01:18:46.731 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.141173.log.gz 2024-09-18T01:18:46.731 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.96613.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52118.log 2024-09-18T01:18:46.731 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.73681.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54095.log 2024-09-18T01:18:46.731 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93232.log.gz 2024-09-18T01:18:46.731 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.52118.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56928.log 2024-09-18T01:18:46.731 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42011.log 2024-09-18T01:18:46.731 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54095.log: /var/log/ceph/cluster1-client.mirror.56928.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48516.log 2024-09-18T01:18:46.731 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.42011.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56510.log 2024-09-18T01:18:46.731 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48516.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47696.log 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56510.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53067.log 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47696.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77708.log 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53067.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28743.log 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.40759.log.gz 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.40812.log.gz 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77708.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37942.log 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28743.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79135.log 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28743.log.gz 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37942.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141562.log 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79135.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105446.log 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.141562.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82934.log 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70335.log 2024-09-18T01:18:46.732 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105446.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.105446.log.gz 2024-09-18T01:18:46.733 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.82934.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64438.log 2024-09-18T01:18:46.733 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.141562.log.gz 2024-09-18T01:18:46.733 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70335.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95524.log 2024-09-18T01:18:46.733 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64438.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.40645.log.gz 2024-09-18T01:18:46.733 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92202.log 2024-09-18T01:18:46.733 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.95524.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51227.log 2024-09-18T01:18:46.733 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92202.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58029.log 2024-09-18T01:18:46.733 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.51227.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57558.log 2024-09-18T01:18:46.734 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58029.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59289.log 2024-09-18T01:18:46.734 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57558.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83257.log 2024-09-18T01:18:46.734 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59289.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131383.log 2024-09-18T01:18:46.734 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.83257.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62442.log 2024-09-18T01:18:46.734 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131383.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102533.log 2024-09-18T01:18:46.734 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.62442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134920.log 2024-09-18T01:18:46.734 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102533.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108670.log 2024-09-18T01:18:46.735 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134920.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115116.log 2024-09-18T01:18:46.735 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.108670.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139002.log 2024-09-18T01:18:46.735 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123295.log 2024-09-18T01:18:46.735 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139002.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103714.log 2024-09-18T01:18:46.735 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50994.log 2024-09-18T01:18:46.735 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123295.log: /var/log/ceph/cluster1-client.mirror.103714.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111893.log 2024-09-18T01:18:46.735 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50994.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115179.log 2024-09-18T01:18:46.736 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50994.log.gz 2024-09-18T01:18:46.736 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92181.log.gz 2024-09-18T01:18:46.736 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111893.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104093.log 2024-09-18T01:18:46.736 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115179.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64802.log 2024-09-18T01:18:46.736 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104093.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81707.log 2024-09-18T01:18:46.736 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64802.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.121162.log 2024-09-18T01:18:46.737 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81707.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84098.log 2024-09-18T01:18:46.737 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.121162.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40739.log 2024-09-18T01:18:46.737 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.84098.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52218.log 2024-09-18T01:18:46.737 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.40739.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36886.log 2024-09-18T01:18:46.737 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33102.log 2024-09-18T01:18:46.737 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.52218.log: /var/log/ceph/cluster1-client.mirror.36886.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34887.log 2024-09-18T01:18:46.738 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33102.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79007.log 2024-09-18T01:18:46.738 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34887.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93802.log 2024-09-18T01:18:46.738 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79007.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69285.log 2024-09-18T01:18:46.738 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.93802.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109767.log 2024-09-18T01:18:46.738 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.93802.log.gz 2024-09-18T01:18:46.738 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34259.log 2024-09-18T01:18:46.739 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69285.log: /var/log/ceph/cluster1-client.mirror.109767.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95959.log 2024-09-18T01:18:46.739 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34259.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133699.log 2024-09-18T01:18:46.739 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95959.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129990.log 2024-09-18T01:18:46.739 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.98975.log.gz 2024-09-18T01:18:46.739 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133699.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140274.log 2024-09-18T01:18:46.739 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.129990.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93399.log 2024-09-18T01:18:46.739 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88665.log 2024-09-18T01:18:46.740 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140274.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140274.log.gz 2024-09-18T01:18:46.740 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93399.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107211.log 2024-09-18T01:18:46.740 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88665.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37599.log 2024-09-18T01:18:46.740 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107211.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108801.log 2024-09-18T01:18:46.740 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37599.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125940.log 2024-09-18T01:18:46.740 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.108801.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110225.log 2024-09-18T01:18:46.741 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.108801.log.gz 2024-09-18T01:18:46.741 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125940.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131500.log 2024-09-18T01:18:46.741 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110225.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83789.log 2024-09-18T01:18:46.741 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.131500.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106838.log 2024-09-18T01:18:46.741 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83789.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33354.log 2024-09-18T01:18:46.741 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106838.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36149.log 2024-09-18T01:18:46.742 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33354.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83723.log 2024-09-18T01:18:46.742 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36149.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112797.log 2024-09-18T01:18:46.742 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster2-client.admin.83723.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.107478.log 2024-09-18T01:18:46.742 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112797.log: 32.2% -- replaced with /var/log/ceph/cluster2-client.admin.84786.log.gz 2024-09-18T01:18:46.742 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116998.log 2024-09-18T01:18:46.742 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.107478.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122099.log 2024-09-18T01:18:46.742 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73476.log 2024-09-18T01:18:46.984 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122099.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74117.log 2024-09-18T01:18:46.985 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73476.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138509.log 2024-09-18T01:18:46.985 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74117.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66886.log 2024-09-18T01:18:46.985 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.138509.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89378.log 2024-09-18T01:18:46.985 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.138509.log.gz 2024-09-18T01:18:46.985 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66886.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67726.log 2024-09-18T01:18:46.985 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89378.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79303.log 2024-09-18T01:18:46.985 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67726.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40832.log 2024-09-18T01:18:46.985 INFO:teuthology.orchestra.run.smithi049.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.86462.log.gz 2024-09-18T01:18:46.985 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79303.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125101.log 2024-09-18T01:18:46.985 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.40832.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107431.log 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125101.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140167.log 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.107431.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53106.log 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.86683.log.gz 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140167.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68227.log 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53106.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91971.log.gzgzip 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44473.log 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64437.log/var/log/ceph/cluster1-client.mirror.68227.log: 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44473.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60994.log 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91740.log 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.60994.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68102.log 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.84318.log.gz 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64437.log: /var/log/ceph/cluster1-client.mirror.91740.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81919.log 2024-09-18T01:18:46.986 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68102.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54614.log 2024-09-18T01:18:46.987 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81919.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82638.log 2024-09-18T01:18:46.987 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54614.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104827.log 2024-09-18T01:18:46.987 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82638.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59709.log 2024-09-18T01:18:46.987 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104827.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87051.log 2024-09-18T01:18:46.987 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59709.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51756.log 2024-09-18T01:18:46.987 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87051.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60570.log 2024-09-18T01:18:46.987 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51756.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58031.log 2024-09-18T01:18:46.987 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60570.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30907.log 2024-09-18T01:18:46.987 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58031.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82581.log 2024-09-18T01:18:46.987 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128697.log 2024-09-18T01:18:46.987 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.30907.log: /var/log/ceph/cluster1-client.admin.82581.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107618.log.gz 2024-09-18T01:18:46.987 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.82581.log.gz 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.95183.log 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.128697.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67177.log 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.95183.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34192.log 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67177.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57985.log 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34192.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40330.log 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57985.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72571.log 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.40330.log: /var/log/ceph/cluster2-client.admin.72571.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78083.log.gz 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43131.log 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52670.log 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43131.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104216.log 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52670.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83202.log 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104216.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69998.log 2024-09-18T01:18:46.988 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.83202.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111639.log 2024-09-18T01:18:46.989 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82297.log.gz 2024-09-18T01:18:46.989 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69998.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101057.log 2024-09-18T01:18:46.989 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.84034.log.gz 2024-09-18T01:18:46.989 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2024-09-18T01:18:46.989 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101057.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106347.log 2024-09-18T01:18:46.989 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49798.log 2024-09-18T01:18:46.989 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.106347.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113111.log 2024-09-18T01:18:46.989 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.49798.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69245.log 2024-09-18T01:18:46.989 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129250.log 2024-09-18T01:18:46.989 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113111.log: /var/log/ceph/cluster1-client.mirror.69245.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116148.log 2024-09-18T01:18:46.989 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129250.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80995.log.gz 2024-09-18T01:18:46.989 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87465.log 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129250.log.gz 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.116148.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43530.log 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.87465.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92116.log 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43530.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51502.log 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34149.log 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58745.log/var/log/ceph/cluster1-client.admin.51502.log: 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34149.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50888.log 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58745.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135557.log 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50888.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95951.log 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50888.log.gz 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135557.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53279.log 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95951.log: gzip -5 --verbose 0.0% -- -- replaced with /var/log/ceph/cluster2-client.admin.95951.log.gz 2024-09-18T01:18:46.990 INFO:teuthology.orchestra.run.smithi049.stderr: /var/log/ceph/cluster1-client.mirror.48368.log 2024-09-18T01:18:46.991 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54974.log.gz 2024-09-18T01:18:46.991 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53279.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141152.log 2024-09-18T01:18:46.991 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48368.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53242.log 2024-09-18T01:18:46.991 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72076.log 2024-09-18T01:18:46.991 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53242.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52195.log 2024-09-18T01:18:46.991 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.72076.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101605.log 2024-09-18T01:18:46.991 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72076.log.gz 2024-09-18T01:18:46.991 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52195.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89125.log 2024-09-18T01:18:46.991 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101605.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115684.log 2024-09-18T01:18:46.991 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89125.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52830.log 2024-09-18T01:18:46.991 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115684.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133700.log 2024-09-18T01:18:46.992 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52830.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79521.log 2024-09-18T01:18:46.992 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133700.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109939.log 2024-09-18T01:18:46.992 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79521.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31025.log 2024-09-18T01:18:46.992 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109939.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138560.log 2024-09-18T01:18:46.992 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59963.log.gz 2024-09-18T01:18:46.992 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.mirror.31025.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.29021.log 2024-09-18T01:18:46.992 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.138560.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.138560.log.gz 2024-09-18T01:18:46.992 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101058.log 2024-09-18T01:18:46.992 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.29021.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107545.log 2024-09-18T01:18:46.992 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29021.log.gz 2024-09-18T01:18:46.992 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.101058.log: -- /var/log/ceph/cluster1-client.mirror.103343.log 2024-09-18T01:18:46.992 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.107545.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109941.log 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.141152.log.gz 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103343.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81372.log 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109941.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83302.log 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81372.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71318.log 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83302.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42512.log 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.71318.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53067.log.gz 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77999.log 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42512.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87874.log 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77999.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57815.log 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87874.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66526.log 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57815.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87940.log 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66526.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84596.log 2024-09-18T01:18:46.993 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87940.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56133.log 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84596.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87358.log 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56133.log: /var/log/ceph/cluster1-client.mirror.87358.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136111.log 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128687.log 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56974.log.gz 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136111.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48838.log 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.128687.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52909.log 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128687.log.gz 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48838.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40718.log 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125813.log 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.40718.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62536.log 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125813.log: 66.8%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.73775.log.gz -5 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.81456.log 2024-09-18T01:18:46.994 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.62536.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89865.log 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81456.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67178.log 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89865.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116169.log 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67178.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94699.log 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116169.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109041.log 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.116169.log.gz 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.94699.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78923.log 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109041.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52324.log 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78923.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47207.log 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52324.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89653.log 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47207.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97535.log 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81958.log 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89653.log: /var/log/ceph/cluster1-client.admin.97535.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48996.log 2024-09-18T01:18:46.995 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81958.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92600.log 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48996.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41626.log 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92600.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100470.log 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41626.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49213.log 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100470.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34258.log 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.49213.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100052.log 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34258.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90771.log 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100052.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101144.log 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72823.log.gz 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90771.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50889.log 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101144.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76567.log 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50889.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105105.log 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50889.log.gz 2024-09-18T01:18:46.996 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76567.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57619.log 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104742.log 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.105105.log: /var/log/ceph/cluster1-client.mirror.57619.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52004.log 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.mirror.104742.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78377.log 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.52004.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117019.log 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78377.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103501.log 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.117019.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136404.log 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103501.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89506.log 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136404.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114486.log 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89506.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48995.log 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114486.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102407.log 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.48995.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103040.log 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48995.log.gz 2024-09-18T01:18:46.997 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89298.log.gz 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102407.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80194.log 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103040.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66609.log 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80194.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50351.log 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66609.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131678.log 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50351.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51616.log 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.131678.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127483.log 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.51616.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33689.log 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127483.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80365.log 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33689.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50972.log 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80365.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41081.log 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50972.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51683.log 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50972.log.gz 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41081.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68568.log 2024-09-18T01:18:46.998 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.51683.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71379.log 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68568.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41960.log 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.71379.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48979.log 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41960.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54973.log 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.48979.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113960.log 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48979.log.gz 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54973.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42575.log 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113960.log: 90.0%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.70335.log.gz 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36108.log 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42575.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92747.log 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36108.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69078.log 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92747.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55688.log 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69078.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139360.log 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55688.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35898.log 2024-09-18T01:18:46.999 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139360.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68653.log 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35898.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125728.log 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68653.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30667.log 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125728.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52622.log 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.30667.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42358.log 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.52622.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95897.log 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.52622.log.gz 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42358.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127783.log 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.95897.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50544.log 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.127783.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77455.log 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50544.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56887.log 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58615.log.gz 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77455.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41590.log 2024-09-18T01:18:47.000 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125392.log.gz 2024-09-18T01:18:47.001 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56887.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60116.log 2024-09-18T01:18:47.001 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41590.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96520.log 2024-09-18T01:18:47.001 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.60116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55796.log 2024-09-18T01:18:47.001 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.96520.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65160.log 2024-09-18T01:18:47.001 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55796.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55603.log 2024-09-18T01:18:47.001 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65160.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37345.log 2024-09-18T01:18:47.001 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55603.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43844.log 2024-09-18T01:18:47.001 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37345.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99755.log 2024-09-18T01:18:47.001 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123630.log 2024-09-18T01:18:47.001 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99755.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93899.log 2024-09-18T01:18:47.001 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123630.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80323.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93899.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80784.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr: 67.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51911.log.gz 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124636.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80323.log: /var/log/ceph/cluster1-client.mirror.80784.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123375.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47358.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124636.log: /var/log/ceph/cluster1-client.mirror.123375.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37838.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.47358.log: -- /var/log/ceph/cluster1-client.mirror.90920.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.37838.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37158.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90920.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133741.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110890.log.gz 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37158.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56614.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133741.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136991.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56614.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115956.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136991.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49715.log 2024-09-18T01:18:47.002 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115956.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131385.log 2024-09-18T01:18:47.003 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.49715.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129629.log 2024-09-18T01:18:47.003 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131385.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103772.log 2024-09-18T01:18:47.003 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129629.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29396.log 2024-09-18T01:18:47.003 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129629.log.gz 2024-09-18T01:18:47.003 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103772.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44304.log 2024-09-18T01:18:47.003 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.29396.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98996.log 2024-09-18T01:18:47.003 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29396.log.gz 2024-09-18T01:18:47.003 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44304.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67975.log 2024-09-18T01:18:47.003 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.98996.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88958.log 2024-09-18T01:18:47.003 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.98996.log.gz 2024-09-18T01:18:47.003 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67975.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53437.log 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88958.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40058.log 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53437.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137246.log 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.40058.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58826.log 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.40058.log.gz 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137246.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33142.log 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46512.log.gz 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58826.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79388.log 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33142.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73657.log 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79388.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93966.log 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73657.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137660.log 0.0% 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.73657.log.gz 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.93966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36528.log 2024-09-18T01:18:47.004 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137660.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47104.log 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36528.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117043.log 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47104.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58322.log 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.117043.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39385.log 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58322.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141494.log 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39385.log: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.107887.log.gz 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118176.log 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.39385.log.gz 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141494.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82254.log 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118176.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100890.log 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82254.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106442.log 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100890.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130721.log 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122218.log 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130721.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129721.log 2024-09-18T01:18:47.005 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122218.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34950.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.129721.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94805.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34950.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54009.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94805.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101477.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54009.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70545.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101477.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42356.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70545.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44472.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42356.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31696.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44472.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53069.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31696.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90009.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45671.log.gz 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53069.log: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.141494.log.gz 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103966.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90009.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50101.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43048.log 2024-09-18T01:18:47.006 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103966.log: /var/log/ceph/cluster2-client.admin.50101.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50101.log.gz 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89799.log 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43048.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70947.log 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89799.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87143.log 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70947.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45522.log 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87143.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142260.log 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45522.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108188.log 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73502.log.gz 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.142260.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142260.log.gz 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.108188.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70502.log 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98807.log 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70502.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87357.log 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.98807.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54404.log 2024-09-18T01:18:47.007 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87357.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125874.log 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54404.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77977.log 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125874.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62310.log 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77977.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136406.log 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.62310.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107410.log 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62310.log.gz 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136406.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94932.log 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.107410.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65246.log 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94932.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46262.log 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65246.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120250.log 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46262.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38493.log 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67391.log.gz 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120250.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35265.log 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67449.log.gz 2024-09-18T01:18:47.008 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.38493.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94011.log 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35265.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63240.log 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.94011.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96862.log 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.63240.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88245.log 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.96862.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107201.log 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88245.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85934.log 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107201.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51890.log 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85934.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35900.log 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51890.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84195.log 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35900.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69917.log 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.84195.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45461.log 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33584.log.gz 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69917.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73358.log 2024-09-18T01:18:47.009 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45461.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108668.log 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73358.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51072.log 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.108668.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.108668.log.gz 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41322.log 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51072.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62206.log 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41322.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120223.log 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.62206.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91194.log 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77767.log.gz 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120223.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51160.log 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91194.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94809.log 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.51160.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96061.log 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94809.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97778.log 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.96061.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107329.log 2024-09-18T01:18:47.010 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97778.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69136.log 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107329.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107329.log.gzgzip 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135730.log 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69136.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99967.log 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135730.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139752.log 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99967.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116934.log 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139752.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.139752.log.gz 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34467.log 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116934.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114909.log 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34467.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58929.log 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35622.log 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58929.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63354.log 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112797.log.gz 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125184.log 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35622.log: /var/log/ceph/cluster1-client.admin.63354.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129267.log 2024-09-18T01:18:47.011 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125184.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120663.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129267.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95737.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.112902.log.gz 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120663.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103251.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95737.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32997.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103251.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48409.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.mirror.32997.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77005.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48409.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123440.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68821.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77005.log: /var/log/ceph/cluster1-client.mirror.123440.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82381.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139208.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68821.log: /var/log/ceph/cluster1-client.mirror.82381.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75026.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139208.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109353.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.75026.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112167.log 2024-09-18T01:18:47.012 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109353.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34402.log 2024-09-18T01:18:47.013 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112167.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39401.log 2024-09-18T01:18:47.013 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34402.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59500.log 2024-09-18T01:18:47.013 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39401.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103586.log 2024-09-18T01:18:47.013 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67818.log 2024-09-18T01:18:47.013 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59500.log: /var/log/ceph/cluster1-client.mirror.103586.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116607.log 2024-09-18T01:18:47.013 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67818.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92245.log 2024-09-18T01:18:47.013 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116607.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77873.log 2024-09-18T01:18:47.013 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92245.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100637.log 2024-09-18T01:18:47.013 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77873.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87095.log 2024-09-18T01:18:47.013 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100637.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84225.log 2024-09-18T01:18:47.013 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.87095.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77623.log 2024-09-18T01:18:47.013 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.84225.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49165.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77623.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116000.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.49165.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126990.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.116000.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125476.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126990.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33519.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125476.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74459.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33519.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126862.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74459.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104426.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66055.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126862.log: /var/log/ceph/cluster1-client.mirror.104426.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99797.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66055.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104721.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99797.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46734.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104721.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51817.log 2024-09-18T01:18:47.014 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37092.log 2024-09-18T01:18:47.015 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.51817.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117349.log 2024-09-18T01:18:47.015 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124889.log.gz 2024-09-18T01:18:47.015 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37092.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35940.log 2024-09-18T01:18:47.015 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117349.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31687.log 2024-09-18T01:18:47.015 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35940.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133366.log 2024-09-18T01:18:47.015 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31687.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137030.log 2024-09-18T01:18:47.015 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133366.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48619.log 2024-09-18T01:18:47.015 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137030.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138126.log 2024-09-18T01:18:47.015 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48619.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108812.log 2024-09-18T01:18:47.015 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40074.log 2024-09-18T01:18:47.015 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.108812.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57610.log 2024-09-18T01:18:47.015 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.40074.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86549.log 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57610.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129022.log 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130973.log 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.129022.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42917.log 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.130973.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72118.log 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42917.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65475.log 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72118.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138505.log 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65475.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71314.log 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.138505.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44622.log 0.0% 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.138505.log.gz 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71314.log: gzip 0.0% -5 -- replaced with /var/log/ceph/cluster1-client.admin.71314.log.gz --verbose 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr: -- /var/log/ceph/cluster1-client.mirror.80069.log 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44622.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132940.log 2024-09-18T01:18:47.016 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80069.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25384.log 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132940.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116144.log 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.25384.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32788.log 0.0% 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.25384.log.gz 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116144.log: gzip 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.116144.log.gz -5 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.88916.log 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131912.log 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88916.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98888.log 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131912.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31604.log 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98888.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75950.log 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31604.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135472.log 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75950.log: gzip 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.105489.log.gz -5 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr: --verbose -- /var/log/ceph/cluster1-client.admin.75307.log 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135472.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39369.log 2024-09-18T01:18:47.017 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.105637.log.gz 2024-09-18T01:18:47.018 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.105640.log.gz 2024-09-18T01:18:47.018 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.75307.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115262.log 2024-09-18T01:18:47.018 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.75307.log.gz 2024-09-18T01:18:47.018 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.39369.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62857.log 2024-09-18T01:18:47.018 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115262.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80950.log 2024-09-18T01:18:47.018 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109120.log 2024-09-18T01:18:47.018 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.62857.log: /var/log/ceph/cluster1-client.mirror.80950.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118463.log 2024-09-18T01:18:47.018 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.109120.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93729.log 2024-09-18T01:18:47.018 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118463.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105599.log 2024-09-18T01:18:47.018 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.93729.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.29977.log 2024-09-18T01:18:47.018 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105599.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115537.log 2024-09-18T01:18:47.018 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.0.29977.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130807.log 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115537.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111683.log 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.130807.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69580.log 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.127939.log.gz 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111683.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36781.log 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69580.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114001.log 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36781.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111326.log 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114001.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92246.log 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111326.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111262.log 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92246.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77557.log 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111262.log: 19.3% -- replaced with /var/log/ceph/cluster1-client.admin.131491.log.gz 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.129668.log 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.131468.log.gz 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77557.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140682.log 2024-09-18T01:18:47.019 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.129668.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124739.log 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140682.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129269.log 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140682.log.gz 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124739.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90136.log 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129269.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52738.log 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90136.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132407.log 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52738.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68736.log 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132407.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38797.log 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68736.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46641.log 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.38797.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112650.log 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46641.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126447.log 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112650.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42154.log 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126447.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85961.log 2024-09-18T01:18:47.020 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.42154.log: gzip -5 0.0% --verbose -- -- replaced with /var/log/ceph/cluster1-client.admin.42154.log.gz 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr: /var/log/ceph/cluster2-client.mirror.96821.log 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85961.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111537.log 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.96821.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125099.log.gz 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65585.log 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.125101.log.gz 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111537.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125100.log.gz 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46305.log 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65585.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124048.log 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46305.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74707.log 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124048.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137662.log 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.74707.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64021.log 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127460.log 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123295.log.gz 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64021.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51730.log 2024-09-18T01:18:47.021 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127460.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57138.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29720.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.51730.log: /var/log/ceph/cluster1-client.mirror.57138.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136195.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.29720.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138193.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136195.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138317.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138193.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126782.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138317.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55645.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126782.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134922.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55645.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43191.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134922.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137240.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43191.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56257.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.128580.log.gz 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137240.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73848.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56257.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93680.log 2024-09-18T01:18:47.022 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.73848.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97395.log 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.93680.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61823.log 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.93680.log.gz 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.97395.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30340.log 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.97395.log.gz 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.61823.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100176.log 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.30340.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118766.log 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100176.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72009.log 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.118766.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131817.log 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72009.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126653.log 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96821.log.gz 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.123631.log.gz 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131817.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60678.log 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126653.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128177.log 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.60678.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77203.log 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.128177.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135769.log 2024-09-18T01:18:47.023 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77203.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95243.log 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135769.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.116880.log 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95243.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46011.log 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.116880.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82727.log 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46011.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124529.log 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.82727.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.82727.log.gzgzip 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107752.log 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44118.log 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.107752.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54571.log 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44118.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123839.log 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54571.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46683.log 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123839.log: 87.4%gzip -- replaced with /var/log/ceph/cluster2-client.admin.83606.log.gz -5 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr: --verbose -- /var/log/ceph/cluster1-client.admin.118813.log 2024-09-18T01:18:47.024 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46683.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61375.log 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.118813.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35941.log 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.61375.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105095.log 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35941.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76873.log 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.105095.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47993.log 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76873.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84570.log 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47993.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76170.log 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84570.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123144.log 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76170.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90726.log 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106529.log 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125813.log.gz 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90726.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57308.log 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106529.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84426.log 2024-09-18T01:18:47.025 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57308.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33215.log 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.84426.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123059.log 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47882.log 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123059.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83249.log 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47882.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110504.log 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.83249.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121582.log 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110504.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42394.log 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121582.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111475.log 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42394.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55748.log 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111475.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58786.log 67.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130560.log.gz 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55748.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39270.log 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58786.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39262.log 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39270.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102700.log 2024-09-18T01:18:47.026 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39262.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117357.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102700.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29840.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117357.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80449.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.29840.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79561.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80449.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135689.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79561.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135555.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135689.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35409.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50104.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35409.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101687.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50104.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135557.log.gz 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48369.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101687.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115119.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48369.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43109.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115119.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57689.log 2024-09-18T01:18:47.027 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43109.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59769.log 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57689.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31129.log 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59769.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107176.log 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.31129.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31129.log.gz 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107846.log 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139239.log 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.107846.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101310.log 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139239.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114381.log 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101310.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41014.log 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114381.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110889.log 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114171.log 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41014.log: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.119336.log.gz 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110889.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36152.log 2024-09-18T01:18:47.028 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114171.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56972.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36152.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101478.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50842.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56972.log: /var/log/ceph/cluster1-client.mirror.101478.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43049.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.50842.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89235.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43049.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119497.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89235.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73962.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.119497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41550.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.73962.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99201.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41550.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131844.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99201.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41128.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.131844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93684.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41128.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115809.log 2024-09-18T01:18:47.029 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93684.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82552.log 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115809.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88454.log 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.82552.log: gzip 0.0% -5 -- replaced with /var/log/ceph/cluster1-client.admin.82552.log.gz --verbose 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr: -- /var/log/ceph/cluster1-client.mirror.122376.log 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88454.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81286.log 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122376.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.95897.log.gz 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133867.log 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81286.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41958.log 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133867.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91133.log 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124765.log 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41958.log: /var/log/ceph/cluster1-client.mirror.91133.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67095.log 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124765.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137242.log 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67095.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125055.log 2024-09-18T01:18:47.030 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137242.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50005.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125055.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63803.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50005.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53380.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.63803.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101226.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.53380.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34779.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101226.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100933.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34779.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114759.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100933.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63525.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70337.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.63525.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.63525.log.gz 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111099.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70337.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85323.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111099.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76811.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.85323.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76853.log 2024-09-18T01:18:47.031 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76811.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83301.log 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76853.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120214.log 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83301.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33142.log.gz 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57474.log 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.33102.log.gz 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120214.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105270.log 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57474.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126655.log 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105270.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100049.log 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.105270.log.gz 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126655.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41040.log 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100049.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104871.log 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.41040.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68611.log 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104871.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139359.log 2024-09-18T01:18:47.032 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68611.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52045.log 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139359.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74734.log 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.52045.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131048.log 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.74734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130539.log 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72628.log 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.130539.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74889.log 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.85961.log.gz 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.72628.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85095.log 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.74889.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61495.log 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.85095.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87104.log 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.61495.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63705.log 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.41322.log.gz 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.87104.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133321.log 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.63705.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69413.log 2024-09-18T01:18:47.033 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133321.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133532.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69413.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78924.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133532.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65129.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78924.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45354.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65129.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141860.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45354.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97503.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.141860.log: 67.3%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.141860.log.gz 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126151.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111262.log.gz 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97503.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68713.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126151.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100891.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68713.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40853.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100891.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107315.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.40853.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59878.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107315.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107315.log.gz 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88455.log 2024-09-18T01:18:47.034 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59878.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90641.log 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88455.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52347.log 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.63354.log.gz 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90641.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56822.log 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.52347.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98611.log 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56822.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40188.log 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98611.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30644.log 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.40188.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140278.log 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.30644.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72032.log 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140278.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105905.log 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72032.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59290.log 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105905.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117700.log 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59290.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97318.log 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56133.log.gz 2024-09-18T01:18:47.035 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117700.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54361.log 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97318.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77790.log 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54361.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30581.log 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77790.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34109.log 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.30581.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102107.log 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34109.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131669.log 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51007.log 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57985.log.gz 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.131669.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68863.log 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.51007.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.51007.log.gz 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34685.log 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68863.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108679.log 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34685.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136571.log 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.108679.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110314.log 2024-09-18T01:18:47.036 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136571.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101815.log 2024-09-18T01:18:47.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55645.log.gz 2024-09-18T01:18:47.037 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110314.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100259.log 2024-09-18T01:18:47.037 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101815.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47318.log 2024-09-18T01:18:47.037 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100259.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71250.log 2024-09-18T01:18:47.037 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47318.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128505.log 2024-09-18T01:18:47.037 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71250.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.71250.log.gz 2024-09-18T01:18:47.037 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30438.log 2024-09-18T01:18:47.037 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.128505.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95681.log 2024-09-18T01:18:47.037 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.30438.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90559.log 2024-09-18T01:18:47.037 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95681.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55191.log 2024-09-18T01:18:47.037 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37820.log.gz 2024-09-18T01:18:47.037 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37838.log.gz 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90559.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78043.log 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.40698.log.gz 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55191.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125369.log 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78043.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53173.log 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.37842.log.gz 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125369.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75817.log.gz 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88959.log 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.53173.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31266.log 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.53173.log.gz 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88959.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116181.log 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31266.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77580.log 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.116181.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46786.log 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77580.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78505.log 2024-09-18T01:18:47.038 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46786.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121784.log 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78505.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55748.log.gz 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77285.log 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121784.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106214.log 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77285.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56342.log 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.106214.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109902.log 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56342.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135051.log 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107333.log 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.109902.log: /var/log/ceph/cluster1-client.mirror.135051.log: 88.8%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.52670.log.gz -5 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr: --verbose -- /var/log/ceph/cluster1-client.admin.38253.log 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107333.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108249.log 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107333.log.gz 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.38253.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120729.log 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.108249.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131754.log 2024-09-18T01:18:47.039 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120729.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33982.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131754.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83949.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33982.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35369.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83949.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73555.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35369.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116779.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73555.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98417.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116779.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59375.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.69998.log.gz 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.140214.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59375.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79858.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.140214.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61689.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79858.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80278.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61689.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85022.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80278.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31357.log 2024-09-18T01:18:47.040 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85022.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122788.log 2024-09-18T01:18:47.041 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31357.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98392.log 2024-09-18T01:18:47.041 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.72009.log.gz 2024-09-18T01:18:47.041 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47248.log 2024-09-18T01:18:47.041 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98392.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79177.log 2024-09-18T01:18:47.041 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47248.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68822.log 2024-09-18T01:18:47.041 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79177.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93022.log 2024-09-18T01:18:47.041 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68822.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61680.log 2024-09-18T01:18:47.041 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93022.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54806.log 2024-09-18T01:18:47.041 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61680.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110060.log 2024-09-18T01:18:47.041 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54806.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75606.log 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110060.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87804.log 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75606.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126024.log 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87804.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142208.log 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126024.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74687.log 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.142208.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70947.log.gz 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126361.log 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142208.log.gz 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74687.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90685.log 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126361.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59792.log 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90685.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93167.log 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59792.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36633.log 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93167.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45001.log 2024-09-18T01:18:47.042 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.74707.log.gz 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118171.log 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45001.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110289.log 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118171.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72468.log 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110289.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94957.log 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72468.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119946.log 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94957.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99673.log 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119946.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122578.log 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99673.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88874.log 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122578.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.116867.log 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88874.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40531.log 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.116867.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50701.log 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.116867.log.gz 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.40531.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133489.log 2024-09-18T01:18:47.043 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50701.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50317.log 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133489.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55689.log 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101100.log.gz 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50317.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66484.log 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50317.log.gz 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34109.log.gz 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55689.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28895.log 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66484.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35516.log 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28895.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28895.log.gz 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132255.log 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35516.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83951.log 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132255.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78715.log 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83951.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.132430.log 2024-09-18T01:18:47.044 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78715.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41456.log 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.132430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83397.log 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41456.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.21613.log 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83397.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137450.log 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/ceph.tmp-client.admin.21613.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.21613.log.gz 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89126.log 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137450.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123839.log.gz 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57054.log 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114127.log 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57054.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112020.log 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114127.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75128.log 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112020.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88328.log 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75128.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30974.log 2024-09-18T01:18:47.045 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88328.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66483.log 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.30974.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.107565.log 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30974.log.gz 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66483.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104764.log 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54571.log.gz 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.107565.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69662.log 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104764.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31656.log 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56468.log 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31656.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94719.log 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56468.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56637.log 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94719.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142312.log 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56637.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80698.log 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46786.log.gz 2024-09-18T01:18:47.046 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.142312.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65541.log 2024-09-18T01:18:47.047 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142312.log.gz 2024-09-18T01:18:47.047 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80698.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93788.log 2024-09-18T01:18:47.047 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65541.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117156.log 2024-09-18T01:18:47.047 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.93788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114424.log 2024-09-18T01:18:47.047 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.93788.log.gz 2024-09-18T01:18:47.047 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.117156.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98133.log 2024-09-18T01:18:47.047 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114424.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51844.log 2024-09-18T01:18:47.047 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98133.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127419.log 2024-09-18T01:18:47.047 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.51844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42202.log 2024-09-18T01:18:47.047 INFO:teuthology.orchestra.run.smithi049.stderr:gzip/var/log/ceph/cluster1-client.admin.127419.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119983.log 2024-09-18T01:18:47.047 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.127419.log.gz 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.76853.log.gz 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42202.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74968.log 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78043.log.gz 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119983.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130824.log 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74968.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31217.log 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.130824.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29421.log 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31217.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95686.log 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.29421.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29421.log.gz 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115829.log 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95686.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113859.log 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115829.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141474.log 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113859.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75474.log 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141474.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103125.log 2024-09-18T01:18:47.048 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.75474.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.141474.log.gz 2024-09-18T01:18:47.049 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103252.log 2024-09-18T01:18:47.049 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103125.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100891.log.gz 2024-09-18T01:18:47.049 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100933.log.gz 2024-09-18T01:18:47.049 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87141.log 2024-09-18T01:18:47.049 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103252.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81329.log 2024-09-18T01:18:47.049 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87141.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62166.log 2024-09-18T01:18:47.049 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81329.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36990.log 2024-09-18T01:18:47.049 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.62166.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137877.log 2024-09-18T01:18:47.049 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36990.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93441.log 2024-09-18T01:18:47.049 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137877.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122639.log 2024-09-18T01:18:47.049 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93441.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104258.log 2024-09-18T01:18:47.049 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46734.log.gz 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122639.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125603.log 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104258.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65844.log 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125603.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69622.log 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47569.log 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69622.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96271.log 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47569.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134247.log 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.96271.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84637.log 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134247.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63745.log 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84637.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61826.log 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.63745.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55794.log 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.61826.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112059.log 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55794.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.105905.log.gz 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70522.log 2024-09-18T01:18:47.050 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112059.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36737.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70522.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45460.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36737.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112103.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45460.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74573.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112103.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45144.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74573.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130238.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49822.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130238.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112060.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.49822.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90429.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112060.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77663.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90429.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86812.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77663.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49360.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45354.log.gz 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86812.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39068.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.49360.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65182.log 2024-09-18T01:18:47.051 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39068.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86047.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65182.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60614.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86047.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115335.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.60614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49895.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115335.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42927.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.49895.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101354.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42927.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100595.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101354.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83721.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100595.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109584.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83721.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56132.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109584.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38935.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127427.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.38935.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68779.log 2024-09-18T01:18:47.052 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127427.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122849.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68779.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45672.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122849.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123419.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45672.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78422.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123419.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111849.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78422.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45479.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111849.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103082.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45479.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112102.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103082.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88392.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112102.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36927.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88392.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33216.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36927.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73027.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33216.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80488.log 2024-09-18T01:18:47.053 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73027.log: /var/log/ceph/cluster1-client.mirror.80488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106629.log 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83300.log 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.106629.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68146.log 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83300.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140460.log 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68146.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58955.log 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140460.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55644.log 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58955.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95696.log 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55644.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49163.log 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95696.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36782.log.gz 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56844.log 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.49163.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.49163.log.gz 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138192.log 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34047.log 2024-09-18T01:18:47.054 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138192.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133175.log 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34047.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34259.log.gz 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98753.log 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133175.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107170.log 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98753.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93085.log 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107170.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107170.log.gz 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42878.log 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93085.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141018.log 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42878.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92452.log 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141018.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51548.log 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100470.log.gz 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92452.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.141018.log.gz 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76895.log 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51548.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126571.log 2024-09-18T01:18:47.055 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76895.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91132.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126571.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139666.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100303.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.139666.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101395.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100303.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29669.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101395.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102957.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.29669.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97444.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102957.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104659.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97444.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122367.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104659.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.30907.log.gz 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28817.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122367.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100009.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28817.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28817.log.gz 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136361.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100009.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45250.log 2024-09-18T01:18:47.056 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136361.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52782.log 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45250.log: gzip -5 89.9% --verbose -- -- replaced with /var/log/ceph/cluster1-client.mirror.35729.log.gz /var/log/ceph/cluster2-client.admin.95369.log 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.52782.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102999.log 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95369.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43697.log 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102999.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93549.log 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43697.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134671.log 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55923.log 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134671.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88606.log 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55923.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77873.log.gz 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73304.log 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88606.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79817.log 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.73304.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131667.log 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79817.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72031.log 2024-09-18T01:18:47.057 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.131667.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.131667.log.gz 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42834.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72031.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106603.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42834.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68358.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.106603.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92746.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68358.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76512.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92746.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114969.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54973.log.gz 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76512.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31131.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114969.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97641.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.31131.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133576.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97641.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51113.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133576.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104427.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.51113.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137035.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104427.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37093.log 2024-09-18T01:18:47.058 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137035.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130867.log 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37093.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60939.log 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130867.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63959.log 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.60939.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87268.log 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.63959.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76458.log 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.83302.log.gz 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.87268.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.87268.log.gz 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122577.log 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.76458.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.76458.log.gz 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83081.log 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122577.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31759.log 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.83081.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78168.log 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.31759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120084.log 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78168.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104785.log 2024-09-18T01:18:47.059 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120084.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97823.log 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104785.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82001.log 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97823.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34362.log 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82001.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75376.log 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34362.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73644.log 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.60994.log.gz 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75376.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32744.log 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73644.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73644.log.gz 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87184.log 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32744.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141421.log 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87184.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33939.log 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.141421.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84867.log 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.141421.log.gz 2024-09-18T01:18:47.060 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33939.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69916.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84867.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84867.log.gz 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42981.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69916.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81497.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42981.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46430.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34634.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46430.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98051.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42356.log.gz 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34634.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124109.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98051.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131005.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124109.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46998.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.131005.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69556.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106040.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69556.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50588.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106040.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50928.log 2024-09-18T01:18:47.061 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.50588.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58574.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118745.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58574.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127030.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.118745.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124260.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127030.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91763.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75950.log.gz 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124260.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44262.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91763.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113705.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44262.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35099.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113705.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119315.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35099.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105075.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.119315.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59245.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.105075.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95573.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59245.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100091.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.95573.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36991.log 2024-09-18T01:18:47.062 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100091.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128581.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124949.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.128581.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93572.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124949.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35519.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93572.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43275.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35519.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100217.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43275.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90345.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100217.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63633.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90345.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41359.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.114909.log.gz 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.63633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106907.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41359.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45588.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106907.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124006.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45588.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132646.log 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.120324.log.gz 2024-09-18T01:18:47.063 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124006.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136824.log 2024-09-18T01:18:47.064 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132646.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115390.log 2024-09-18T01:18:47.064 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136824.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96365.log 2024-09-18T01:18:47.064 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115390.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83719.log 2024-09-18T01:18:47.064 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.96365.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85987.log 2024-09-18T01:18:47.064 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83719.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.83719.log.gz 2024-09-18T01:18:47.064 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65887.log 2024-09-18T01:18:47.064 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.85987.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91762.log 2024-09-18T01:18:47.064 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65887.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103839.log 2024-09-18T01:18:47.064 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91762.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94057.log 2024-09-18T01:18:47.064 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103839.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113650.log 2024-09-18T01:18:47.064 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.94057.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96174.log 2024-09-18T01:18:47.190 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113650.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43315.log 2024-09-18T01:18:47.190 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.96174.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96174.log.gz 2024-09-18T01:18:47.190 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29891.log 2024-09-18T01:18:47.190 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43315.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60429.log 2024-09-18T01:18:47.190 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.29891.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124763.log.gz 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124765.log.gz 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52791.log 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60429.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60429.log.gz 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55351.log 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.52791.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35942.log 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55351.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49280.log 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35942.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59414.log 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.49280.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.123144.log.gz 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80154.log 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59414.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45947.log 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80154.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46009.log 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45947.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43023.log 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46009.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90075.log 2024-09-18T01:18:47.191 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43023.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47695.log 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90075.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83780.log 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47695.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69666.log 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125369.log.gz 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83780.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75483.log 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69666.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77834.log 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.75483.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87021.log 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77834.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95538.log 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87021.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50891.log 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.95538.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53708.log 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50891.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115054.log 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53708.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63012.log 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115054.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53872.log 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.130807.log.gz 2024-09-18T01:18:47.192 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.63012.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123001.log 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53872.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.130824.log.gz 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112483.log 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123001.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64075.log 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112483.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83203.log 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64075.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62837.log 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.83203.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40038.log 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.62837.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.125055.log.gz 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59139.log 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.40038.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135344.log 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59139.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90767.log 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135344.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71221.log 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90767.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69623.log 2024-09-18T01:18:47.193 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71221.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57183.log 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.71221.log.gz 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69623.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105168.log 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57183.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48863.log 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126655.log.gz 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.105168.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126023.log 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.48863.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52486.log 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126023.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33626.log 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.52486.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39688.log 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33626.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61975.log 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39688.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135261.log 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.39688.log.gz 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61975.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.61975.log.gz 2024-09-18T01:18:47.194 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47948.log 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135261.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101774.log 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47948.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141193.log 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101774.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.126024.log.gz 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80320.log 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.141193.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76274.log 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.141193.log.gz 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80320.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91615.log 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76274.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83945.log 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91615.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91065.log 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83945.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.83945.log.gz 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93487.log 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91065.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44203.log 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93487.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84888.log 2024-09-18T01:18:47.195 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44203.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69977.log 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84888.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84888.log.gz 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92665.log 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69977.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73347.log 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92665.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124051.log 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73347.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101564.log 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73347.log.gz 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124051.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74324.log 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101564.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114212.log 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.74324.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55015.log 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114212.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124973.log 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55015.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86028.log 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124973.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105196.log 2024-09-18T01:18:47.196 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86028.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61977.log 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.105196.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126630.log 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61977.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91824.log 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.121003.log.gz 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126630.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66759.log 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91824.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80572.log 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62670.log 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80572.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.121132.log.gz 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116323.log 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.62670.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57393.log 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.116323.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141266.log 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57393.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93086.log 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141266.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63842.log 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93086.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62225.log 2024-09-18T01:18:47.197 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.63842.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114842.log 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.62225.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76614.log 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114842.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69242.log 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76614.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109005.log 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69242.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119242.log 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109005.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105193.log 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.119242.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40045.log 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105193.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.105193.log.gz 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41693.log 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.40045.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.40045.log.gz 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120750.log 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41693.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38694.log 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120750.log: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.141266.log.gz 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.120750.log.gz 2024-09-18T01:18:47.198 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43355.log 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.38694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51984.log 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43355.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79604.log 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51984.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54721.log 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79604.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63334.log 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54721.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109514.log 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.63334.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96161.log 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.109514.log: gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.96540.log.gz 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.96161.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96161.log.gz 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.93899.log.gz 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54360.log 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96707.log.gz 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139890.log 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54360.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81834.log 2024-09-18T01:18:47.199 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139890.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133909.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81834.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89717.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133909.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84354.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89717.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86571.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84354.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60447.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.86571.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.86571.log.gz 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115683.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.60447.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105009.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115683.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113704.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105009.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63575.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113704.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50419.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.63575.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132939.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.50419.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71864.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132939.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73982.log 2024-09-18T01:18:47.200 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71864.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.97444.log.gz 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102362.log 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.73982.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94807.log 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102362.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133828.log 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94807.log: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.31357.log.gz 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97337.log 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133828.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108208.log 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97337.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118545.log 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.108208.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130151.log 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118545.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118545.log.gz 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85163.log 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130151.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72117.log 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85163.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.83203.log.gz 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44791.log 2024-09-18T01:18:47.201 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72117.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120351.log 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44791.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28821.log 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120351.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130519.log 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28821.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28821.log.gz 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35160.log 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.130519.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138834.log 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35160.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90429.log.gz 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105980.log 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.138834.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53371.log 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.105980.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.105980.log.gz 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127526.log 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.53371.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79480.log 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127526.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52072.log 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79480.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90559.log.gz 2024-09-18T01:18:47.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94514.log 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.52072.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79008.log 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94514.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.85531.log.gz 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134038.log 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79008.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135346.log 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134038.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73594.log 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135346.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70186.log 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73594.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135773.log 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70186.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100847.log 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135773.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98657.log 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100847.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125475.log 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89378.log.gz 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98657.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63313.log 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125475.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31140.log 2024-09-18T01:18:47.203 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.63313.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83924.log 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.31140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130425.log 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.83924.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119678.log 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89506.log.gz 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.130425.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53176.log 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119678.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51092.log 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61036.log 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51092.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78336.log 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61036.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47568.log 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93441.log.gz 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78336.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78545.log 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47568.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28813.log 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78545.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113532.log 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28813.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28813.log.gz 2024-09-18T01:18:47.204 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76040.log 2024-09-18T01:18:47.205 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113532.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37599.log.gz 2024-09-18T01:18:47.205 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104681.log 2024-09-18T01:18:47.205 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.76040.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141360.log 2024-09-18T01:18:47.205 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104681.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67137.log 2024-09-18T01:18:47.205 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141360.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107185.log 2024-09-18T01:18:47.205 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67137.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.141360.log.gz 2024-09-18T01:18:47.205 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55540.log 2024-09-18T01:18:47.205 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.94719.log.gz 2024-09-18T01:18:47.205 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107185.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.94807.log.gz 2024-09-18T01:18:47.205 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113365.log 2024-09-18T01:18:47.205 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55540.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89655.log 2024-09-18T01:18:47.205 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113365.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33269.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55644.log.gz 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89655.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126152.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33269.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29601.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126152.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73822.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.29601.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127669.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.73822.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31902.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.127669.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97803.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31902.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117457.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97803.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122016.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117457.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139003.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122016.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88061.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139003.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68610.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88061.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39693.log 2024-09-18T01:18:47.206 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68610.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108322.log 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39693.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57559.log 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.108322.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73754.log 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57559.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95904.log 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.73754.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46156.log 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.95904.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135770.log 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46156.log: gzip -5 --verbose -- /var/log/ceph/cluster2.log 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135770.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135771.log 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124049.log 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.105009.log.gz 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135771.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37863.log 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56132.log.gz 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124049.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95847.log 2024-09-18T01:18:47.207 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37863.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135729.log 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95847.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112904.log 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135729.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138633.log 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112904.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101688.log 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.138633.log: 0.0%gzip -- replaced with /var/log/ceph/cluster1-client.admin.138633.log.gz 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.121369.log 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101688.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92160.log 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100217.log.gz 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.121369.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125686.log 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92160.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118549.log 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125686.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37817.log 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87941.log 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118549.log.gz 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.37817.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37817.log.gz 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101522.log 2024-09-18T01:18:47.208 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87941.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102492.log 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.63012.log.gz 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101522.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124469.log 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102492.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42462.log 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124469.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109893.log 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.42462.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42462.log.gz 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101939.log 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.109893.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86900.log 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101939.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137244.log 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.86900.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95671.log 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137244.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40028.log 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95671.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.95671.log.gz 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133618.log 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.40028.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80364.log 2024-09-18T01:18:47.209 INFO:teuthology.orchestra.run.smithi049.stderr: 88.1% -- replaced with /var/log/ceph/cluster2.log.gz 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133618.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32679.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80364.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68205.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32679.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89443.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68205.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63656.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89443.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133550.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53555.log.gz 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.63656.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46429.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133550.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65969.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46429.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126107.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65969.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124975.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126107.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99714.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124975.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115056.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99714.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110697.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115056.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82610.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110697.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73009.log 2024-09-18T01:18:47.210 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.82610.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.82610.log.gz 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84797.log 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73009.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73009.log.gz 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.76040.log.gz 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86416.log 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84797.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91824.log.gz 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70125.log 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.86416.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49116.log 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70125.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72441.log 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.49116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29296.log 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72441.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122368.log 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.29296.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29296.log.gz 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102233.log 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122368.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124268.log 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102233.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86157.log 2024-09-18T01:18:47.211 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124268.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65626.log 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86157.log: gzip -5 89.8% --verbose -- -- replaced with /var/log/ceph/cluster1-client.mirror.35409.log.gz /var/log/ceph/cluster1-client.mirror.54530.log 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65626.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105219.log 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54530.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92043.log 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.105219.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.105219.log.gz 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42202.log.gz 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50893.log 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92043.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104469.log 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50893.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34686.log 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104469.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141746.log 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34686.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112860.log.gz 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113917.log 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.141746.log: 68.0% -- replaced with /var/log/ceph/cluster1-client.mirror.141746.log.gz 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73025.log 2024-09-18T01:18:47.212 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113917.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99815.log 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73025.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84636.log 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.41040.log.gz 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99815.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.41014.log.gz 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50061.log 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84636.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60100.log 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50061.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50061.log.gz 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42673.log 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60100.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60100.log.gz 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62697.log 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.42673.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42673.log.gz 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64619.log 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.62697.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120822.log 2024-09-18T01:18:47.213 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64619.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45421.log 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120822.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138501.log 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45421.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120046.log 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34362.log.gz 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.138501.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.138501.log.gz 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57518.log 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120046.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.120046.log.gz 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35242.log 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57518.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80869.log 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35242.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32421.log 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80869.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83381.log 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32421.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83711.log 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster1-client.admin.75474.log.gz 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83381.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58449.log 2024-09-18T01:18:47.214 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83711.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.83711.log.gz 2024-09-18T01:18:47.215 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44263.log 2024-09-18T01:18:47.215 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58449.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106107.log 2024-09-18T01:18:47.215 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44263.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32113.log 2024-09-18T01:18:47.215 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.106107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129835.log 2024-09-18T01:18:47.215 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32113.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140846.log 2024-09-18T01:18:47.215 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.129835.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122098.log 2024-09-18T01:18:47.215 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140846.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140846.log.gz 2024-09-18T01:18:47.215 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86898.log 2024-09-18T01:18:47.215 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122098.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118239.log 2024-09-18T01:18:47.215 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.86898.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.86898.log.gz 2024-09-18T01:18:47.215 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34844.log 2024-09-18T01:18:47.215 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118239.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130788.log 2024-09-18T01:18:47.216 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.107337.log 2024-09-18T01:18:47.216 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65246.log.gz 2024-09-18T01:18:47.216 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.130788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137161.log 2024-09-18T01:18:47.216 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.107337.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.94775.log 2024-09-18T01:18:47.216 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137161.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55394.log 2024-09-18T01:18:47.216 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.94775.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91193.log 2024-09-18T01:18:47.216 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70606.log.gz 2024-09-18T01:18:47.216 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster1-client.admin.109514.log.gz 2024-09-18T01:18:47.216 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55394.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55183.log 2024-09-18T01:18:47.216 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91193.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113577.log 2024-09-18T01:18:47.216 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55183.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53962.log 2024-09-18T01:18:47.216 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113577.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106185.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53962.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84736.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106185.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131314.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.84736.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47317.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131314.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81415.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47317.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42310.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81415.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89507.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.42310.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108780.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89507.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56048.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.108780.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31561.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56048.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103123.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.31561.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41826.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103123.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124471.log 2024-09-18T01:18:47.217 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41826.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107125.log 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124471.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29703.log 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.107125.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93167.log.gz 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82127.log 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.29703.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135473.log 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82127.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78378.log 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135473.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127550.log 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78378.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121065.log 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.127550.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101898.log 0.0% 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.127550.log.gz 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121065.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130010.log 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101898.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32046.log 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130010.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131954.log 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32046.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50250.log.gz 2024-09-18T01:18:47.218 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119611.log 2024-09-18T01:18:47.219 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131954.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106281.log 2024-09-18T01:18:47.219 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.50701.log.gz 2024-09-18T01:18:47.219 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.119611.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.50588.log.gz 2024-09-18T01:18:47.219 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74801.log 2024-09-18T01:18:47.219 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.106281.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64174.log 2024-09-18T01:18:47.219 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74801.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112019.log 2024-09-18T01:18:47.219 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64174.log: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.131558.log.gz 2024-09-18T01:18:47.219 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.95175.log 2024-09-18T01:18:47.219 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112019.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96198.log 2024-09-18T01:18:47.219 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.95175.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98244.log 2024-09-18T01:18:47.219 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.96198.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126369.log 2024-09-18T01:18:47.219 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98244.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125276.log 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.51817.log.gz 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126369.log: 90.9% -- replaced with /var/log/ceph/cluster1-client.admin.51844.log.gz 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125034.log 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125276.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102873.log 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125034.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113157.log 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102873.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53348.log 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113157.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139427.log 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53348.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126572.log 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139427.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100092.log 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126572.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41541.log 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100092.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102659.log 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41541.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80489.log 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102659.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97090.log 2024-09-18T01:18:47.220 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80489.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87713.log 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97090.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90010.log 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.87713.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28605.log 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90010.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34004.log 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28605.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28605.log.gz 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87652.log 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34004.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142364.log 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87652.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36107.log 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.142364.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142364.log.gz 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87442.log 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36107.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75202.log 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78713.log 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75202.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37970.log 2024-09-18T01:18:47.221 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78713.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38912.log 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.37970.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37970.log.gz 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125318.log 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.38912.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47361.log 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125318.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79260.log 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47361.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73640.log 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79260.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40873.log 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73640.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73640.log.gz 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98457.log 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.40873.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51046.log 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98457.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65207.log 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.51046.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88815.log 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.65207.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47144.log 2024-09-18T01:18:47.222 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88815.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58155.log 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.61495.log.gz 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47144.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82360.log 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58155.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116283.log 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82360.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37996.log 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.116283.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95962.log 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37996.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88041.log 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95962.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110627.log 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61826.log.gz 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88041.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114548.log 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110627.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123925.log 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114548.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48909.log 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123925.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37862.log 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.48909.log: gzip -5 --verbose 0.0% -- /var/log/ceph/cluster2-client.admin.25360.log -- replaced with /var/log/ceph/cluster1-client.admin.48909.log.gz 2024-09-18T01:18:47.223 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37862.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101309.log 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.25360.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25360.log.gz 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39118.log 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101309.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87711.log 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39118.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126422.log 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.87711.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.87711.log.gz 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107243.log 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69556.log.gz 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68358.log.gz 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126422.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44790.log 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107243.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38613.log 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44790.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112734.log 2024-09-18T01:18:47.224 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.38613.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48116.log 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45882.log 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108934.log 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45882.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28900.log 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.108934.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45986.log 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28900.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28900.log.gz 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45734.log 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45986.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136190.log 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36632.log 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136190.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51015.log 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36632.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107524.log 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.51015.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.51015.log.gz 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110059.log 2024-09-18T01:18:47.225 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.107524.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133287.log 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110059.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53489.log 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46429.log.gz 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133287.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46430.log.gz 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141059.log 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53489.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122449.log 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.141059.log: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.141059.log.gz 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97482.log 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122449.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31900.log 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97482.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140819.log 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31900.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135303.log 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140819.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140819.log.gz 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138602.log 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135303.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73889.log 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138602.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82045.log 2024-09-18T01:18:47.226 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124268.log.gz 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.73889.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129161.log 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82045.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.129923.log 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.129161.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96182.log 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.129923.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34466.log 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.96182.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96182.log.gz 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114170.log 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34466.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41101.log 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114170.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69452.log 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41101.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83875.log 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69452.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32722.log 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.83875.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128684.log 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32722.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113112.log 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124049.log.gz 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.128684.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128684.log.gz 2024-09-18T01:18:47.227 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71225.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113112.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98419.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.71225.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34529.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98419.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119944.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93510.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119944.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48198.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93510.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44621.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48198.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54269.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44621.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88538.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.105196.log.gz 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54269.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119041.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88538.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43489.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.119041.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61155.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43489.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75160.log 2024-09-18T01:18:47.228 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.61155.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118523.log 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.75160.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130914.log 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118523.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118523.log.gz 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97577.log 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.130914.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115746.log 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97577.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.118900.log 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115746.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39684.log 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.118900.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84892.log 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39684.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.39684.log.gz 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86291.log 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84892.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84892.log.gz 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77130.log 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86291.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41874.log 2024-09-18T01:18:47.229 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77130.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92580.log 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41874.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45732.log 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92580.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96024.log 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45732.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81664.log 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.96024.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57517.log 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81664.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61289.log 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57517.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116932.log 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61289.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120048.log 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116932.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50090.log 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120048.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135771.log.gz 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40490.log 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50090.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50090.log.gz 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135773.log.gz 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63484.log 2024-09-18T01:18:47.230 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.40490.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138888.log 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.63484.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34845.log 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.138888.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100427.log 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34845.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68948.log 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100427.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76038.log 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68948.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137161.log.gz 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99500.log 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.76038.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.76038.log.gz 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46513.log 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99500.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89929.log 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46513.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37661.log 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89929.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125244.log 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137246.log.gz 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37661.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58869.log 2024-09-18T01:18:47.231 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125244.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141588.log 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58869.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133448.log 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141588.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29371.log 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.141588.log.gz 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133448.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89925.log 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.29371.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29371.log.gz 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90139.log 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89925.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111809.log 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90139.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110002.log 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111809.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62202.log 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110002.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43907.log 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138192.log.gz 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.62202.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62202.log.gz 2024-09-18T01:18:47.232 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131974.log 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43907.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33101.log 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131974.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112229.log 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33101.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137201.log 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112229.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67725.log 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132939.log.gz 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137201.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52539.log 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67725.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85437.log 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52539.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109851.log 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.85437.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96312.log 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109851.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.39118.log.gz 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136361.log.gz 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73665.log 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.96312.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66273.log 2024-09-18T01:18:47.233 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73665.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73665.log.gz 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67306.log 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66273.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110503.log 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67306.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100386.log 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110503.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50109.log 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100386.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97932.log 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50109.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55226.log 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97932.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42050.log 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55226.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90222.log 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42050.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68654.log 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90222.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45103.log 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68654.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38394.log 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45103.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95395.log 2024-09-18T01:18:47.234 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.38394.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122873.log 2024-09-18T01:18:47.235 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95395.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73936.log 2024-09-18T01:18:47.235 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122873.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45042.log 2024-09-18T01:18:47.235 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.73936.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78254.log 2024-09-18T01:18:47.289 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45042.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82000.log 2024-09-18T01:18:47.289 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78254.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106513.log 2024-09-18T01:18:47.289 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82000.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134921.log 2024-09-18T01:18:47.289 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106513.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36252.log 2024-09-18T01:18:47.289 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134921.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63448.log 2024-09-18T01:18:47.289 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36252.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132602.log.gz 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101015.log 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.63448.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85914.log 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101015.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131853.log 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.85914.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34150.log 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.131853.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80909.log 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34150.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108048.log 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80909.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129637.log 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.108048.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93230.log 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129637.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129637.log.gz 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135983.log 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93230.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75015.log 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135983.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56929.log 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.75015.log: 0.0% 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.75015.log.gz 2024-09-18T01:18:47.290 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.134671.log.gz 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64992.log 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56929.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41321.log 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.64992.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.64992.log.gz 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29908.log 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41321.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53666.log 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.29908.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52933.log 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53666.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33690.log 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.52933.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.52933.log.gz 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110780.log 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33690.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66757.log 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110780.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119811.log 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113365.log.gz 2024-09-18T01:18:47.291 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66757.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.140300.log 2024-09-18T01:18:47.292 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119811.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72143.log 2024-09-18T01:18:47.292 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.140300.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137871.log 2024-09-18T01:18:47.292 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72143.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58659.log 2024-09-18T01:18:47.292 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137871.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136781.log 2024-09-18T01:18:47.292 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113534.log.gz 2024-09-18T01:18:47.292 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113532.log.gz 2024-09-18T01:18:47.292 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58659.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86725.log 2024-09-18T01:18:47.292 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89443.log.gz 2024-09-18T01:18:47.292 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136781.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130493.log 2024-09-18T01:18:47.292 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.86725.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.86725.log.gz 2024-09-18T01:18:47.292 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97999.log 2024-09-18T01:18:47.292 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130493.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89442.log 2024-09-18T01:18:47.293 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.97999.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.97999.log.gz 2024-09-18T01:18:47.293 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37879.log 2024-09-18T01:18:47.293 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102658.log 2024-09-18T01:18:47.293 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37879.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116816.log 2024-09-18T01:18:47.293 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102658.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71561.log 2024-09-18T01:18:47.293 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116816.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98796.log 2024-09-18T01:18:47.293 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71561.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70042.log 2024-09-18T01:18:47.293 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.98796.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.98796.log.gz 2024-09-18T01:18:47.293 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70124.log 2024-09-18T01:18:47.293 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70042.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71260.log 2024-09-18T01:18:47.293 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70124.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64523.log 2024-09-18T01:18:47.293 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71260.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.71260.log.gz 2024-09-18T01:18:47.294 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125812.log 2024-09-18T01:18:47.294 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64523.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72504.log 2024-09-18T01:18:47.294 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125812.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88002.log 2024-09-18T01:18:47.294 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.72504.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112587.log 2024-09-18T01:18:47.294 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88002.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78674.log 2024-09-18T01:18:47.294 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112587.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75414.log 2024-09-18T01:18:47.294 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78674.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119523.log 2024-09-18T01:18:47.294 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75414.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77288.log 2024-09-18T01:18:47.294 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.119523.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.140140.log.gz 2024-09-18T01:18:47.294 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58321.log 2024-09-18T01:18:47.294 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.140120.log.gz 2024-09-18T01:18:47.294 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.86416.log.gz 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77288.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105555.log 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58321.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103797.log 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105555.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81288.log 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103797.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52828.log 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81288.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74646.log 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52828.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44329.log 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.74646.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61017.log 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44329.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139986.log 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61017.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.61017.log.gz 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137410.log 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139986.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135345.log 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137410.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98186.log 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135345.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.95129.log 2024-09-18T01:18:47.295 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.98186.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31825.log 2024-09-18T01:18:47.296 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.95129.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112376.log 2024-09-18T01:18:47.296 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31825.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115476.log 2024-09-18T01:18:47.296 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112376.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74003.log 2024-09-18T01:18:47.296 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115476.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78966.log 2024-09-18T01:18:47.296 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74003.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54092.log 2024-09-18T01:18:47.296 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.47948.log.gz 2024-09-18T01:18:47.296 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100513.log 2024-09-18T01:18:47.296 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54092.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75734.log 2024-09-18T01:18:47.296 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100513.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85417.log 2024-09-18T01:18:47.296 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57944.log 2024-09-18T01:18:47.296 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.85417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113429.log 2024-09-18T01:18:47.296 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57944.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76214.log 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113429.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93897.log 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37996.log.gz 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.76214.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.76214.log.gz 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76126.log 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.93897.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.93897.log.gz 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35662.log 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29221.log 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62328.log 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.29221.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29221.log.gz 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121220.log 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.62328.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76527.log 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121220.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81160.log 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76527.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104300.log 2024-09-18T01:18:47.297 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81160.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57182.log 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104300.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136741.log 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57182.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34425.log 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.108934.log.gz 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88815.log.gz 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136741.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.94514.log.gz 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131800.log 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34425.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103672.log 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100427.log.gz 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131800.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80741.log 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103672.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41695.log 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80741.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127721.log 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81415.log.gz 2024-09-18T01:18:47.298 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41695.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123093.log 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127721.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98890.log 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123093.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88749.log 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98890.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111036.log 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90222.log.gz 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88749.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89022.log 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111036.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124133.log 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.84797.log.gz 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89022.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91342.log 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124133.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94956.log 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57944.log.gz 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91342.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42634.log 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94956.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111894.log 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42634.log: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.86291.log.gz 2024-09-18T01:18:47.299 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58659.log.gz 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106242.log 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111894.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75627.log 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.106242.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87463.log 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75627.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55479.log 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.87463.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.87463.log.gz 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140542.log 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55479.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103419.log 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140542.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92831.log 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103419.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32722.log.gz 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49468.log 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92831.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63293.log 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.49468.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37410.log 2024-09-18T01:18:47.300 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.63293.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111264.log 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37410.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54317.log 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.87941.log.gz 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111264.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135132.log 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54317.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120180.log 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46851.log 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.98244.log.gz 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120180.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98485.log 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46851.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33628.log 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98485.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90768.log 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33628.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114549.log 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90768.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117378.log 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123420.log 2024-09-18T01:18:47.301 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117378.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63654.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123420.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83017.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.63654.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.63654.log.gz 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77579.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.83017.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111433.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77579.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90495.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111433.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35623.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90495.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75840.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35623.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134713.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.75840.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41518.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134713.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115009.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41518.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108851.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115009.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105790.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.108851.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86516.log 2024-09-18T01:18:47.302 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.105790.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59834.log 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86516.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127201.log 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59834.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58662.log 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127201.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54530.log.gz 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126739.log 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137241.log 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126739.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41828.log 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137241.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114882.log 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41828.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29874.log 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114882.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28969.log 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.29874.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76948.log 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28969.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28969.log.gz 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52098.log 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76948.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54487.log 2024-09-18T01:18:47.303 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.52098.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122663.log 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54487.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32070.log 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122663.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95263.log 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32070.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52025.log 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95263.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58744.log 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52025.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.140013.log 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58744.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120372.log 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.140013.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73635.log 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120372.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.120372.log.gz 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32789.log 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73635.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73635.log.gz 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89380.log 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32789.log: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.73025.log.gz 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39882.log 2024-09-18T01:18:47.304 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89380.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135789.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.39882.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51186.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135789.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122999.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51186.log: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.73555.log.gz 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105977.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122999.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45167.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.105977.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.105977.log.gz 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58405.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45167.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44750.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58405.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120886.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44750.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42165.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120886.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42742.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.42165.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133497.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42742.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74438.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97615.log 2024-09-18T01:18:47.305 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.74438.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122427.log 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97615.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40900.log 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122427.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60767.log.gz 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69368.log 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.40900.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127503.log 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69368.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104048.log 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127503.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90983.log 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128715.log 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90983.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108849.log 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.128715.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90496.log 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.108849.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104594.log 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55191.log.gz 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90496.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111600.log 2024-09-18T01:18:47.306 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104594.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30779.log 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111600.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46092.log 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.30779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118631.log 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55351.log.gz 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46092.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76049.log 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.118631.log: gzip 89.9% -5 -- replaced with /var/log/ceph/cluster1-client.mirror.57182.log.gz --verbose 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr: -- /var/log/ceph/cluster1-client.mirror.99327.log 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55394.log.gz 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.76049.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105470.log 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99327.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46574.log 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105470.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128707.log 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46574.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112947.log 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.128707.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70631.log 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112947.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69977.log.gz 2024-09-18T01:18:47.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70208.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70631.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73424.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70208.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92035.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73424.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54850.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92035.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109653.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54850.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43464.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95357.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43464.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.49540.log.gz 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32786.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95357.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52227.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32786.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91192.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.52227.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120884.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91192.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32114.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120884.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73008.log 2024-09-18T01:18:47.308 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32114.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74780.log 2024-09-18T01:18:47.309 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73008.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101563.log 2024-09-18T01:18:47.309 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73008.log.gz 2024-09-18T01:18:47.309 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.74780.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126840.log 2024-09-18T01:18:47.309 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101563.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70252.log 2024-09-18T01:18:47.309 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47317.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.47318.log.gz 2024-09-18T01:18:47.309 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:47.309 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47358.log.gz 2024-09-18T01:18:47.309 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126840.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123269.log 2024-09-18T01:18:47.333 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70252.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134497.log 2024-09-18T01:18:47.333 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123269.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135556.log 2024-09-18T01:18:47.333 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134880.log 2024-09-18T01:18:47.333 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135556.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136834.log 2024-09-18T01:18:47.333 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134880.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92537.log 2024-09-18T01:18:47.333 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136834.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119356.log 2024-09-18T01:18:47.333 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92537.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66715.log 2024-09-18T01:18:47.333 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.119356.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129562.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66715.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138611.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.129562.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110656.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.138611.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.138611.log.gz 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44410.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110656.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76515.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44410.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58198.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76515.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72258.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58198.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131993.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72258.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.64174.log.gz 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99586.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131993.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78129.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99586.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72623.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78129.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121866.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.72623.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38798.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121866.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132514.log 2024-09-18T01:18:47.334 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.38798.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135690.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132514.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30366.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135690.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50068.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.30366.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135343.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50068.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50068.log.gz 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39697.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135343.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121911.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57183.log.gz 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39697.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85893.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121911.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84500.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.85893.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72273.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.84500.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80111.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72273.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116910.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80111.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79901.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116910.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119067.log 2024-09-18T01:18:47.335 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79901.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68145.log 2024-09-18T01:18:47.336 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.119067.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84829.log 2024-09-18T01:18:47.336 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68145.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57307.log 2024-09-18T01:18:47.336 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84829.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141608.log 2024-09-18T01:18:47.336 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57307.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69369.log 2024-09-18T01:18:47.336 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66841.log.gz 2024-09-18T01:18:47.336 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141608.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106444.log 2024-09-18T01:18:47.336 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69369.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62208.log 2024-09-18T01:18:47.336 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106444.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112315.log 2024-09-18T01:18:47.336 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.62208.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112104.log 2024-09-18T01:18:47.336 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112315.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33846.log 2024-09-18T01:18:47.336 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112104.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106368.log 2024-09-18T01:18:47.336 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33846.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106589.log 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.106368.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30871.log 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.106589.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97856.log 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.30871.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106283.log 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.97856.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105903.log 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.106283.log: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.141608.log.gz 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93794.log 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105903.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29171.log 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.93794.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.93794.log.gz 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65044.log 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.29171.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29171.log.gz 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73193.log 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65044.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142286.log 2024-09-18T01:18:47.337 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73193.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118432.log 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.142286.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142286.log.gz 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50339.log 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118432.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118432.log.gz 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60878.log 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50339.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92602.log 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.60878.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47420.log 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92602.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112586.log 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47420.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98637.log 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112586.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89843.log 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98637.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127787.log 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89843.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.118585.log 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119523.log.gz 2024-09-18T01:18:47.338 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.127787.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55838.log 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.118585.log: 19.3% -- replaced with /var/log/ceph/cluster1-client.admin.120048.log.gz 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39408.log 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55838.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64057.log 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39408.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49174.log 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33101.log.gz 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64057.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79218.log 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.49174.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93705.log 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79218.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29560.log 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.109851.log.gz 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.93705.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.93705.log.gz 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65266.log 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.29560.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29560.log.gz 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110199.log 2024-09-18T01:18:47.339 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65266.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130844.log 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110199.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87509.log 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.130844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62898.log 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87509.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118455.log 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.62898.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44094.log 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118455.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43487.log 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44094.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.110103.log 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43487.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90302.log 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.110103.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124471.log.gz 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78587.log 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90302.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73023.log 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78587.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64709.log 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73023.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90558.log 2024-09-18T01:18:47.340 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64709.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64019.log 2024-09-18T01:18:47.341 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90558.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140832.log 2024-09-18T01:18:47.341 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64019.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82233.log 2024-09-18T01:18:47.341 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140832.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140832.log.gz 2024-09-18T01:18:47.341 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83654.log 2024-09-18T01:18:47.341 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137871.log.gz 2024-09-18T01:18:47.341 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82233.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66527.log 2024-09-18T01:18:47.341 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83654.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49941.log 2024-09-18T01:18:47.352 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66527.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40215.log 2024-09-18T01:18:47.352 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.49941.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72817.log 2024-09-18T01:18:47.352 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.40215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66675.log 2024-09-18T01:18:47.352 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.72817.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55690.log 2024-09-18T01:18:47.352 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66675.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107732.log 2024-09-18T01:18:47.352 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55690.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59035.log 2024-09-18T01:18:47.352 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.107732.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42000.log 2024-09-18T01:18:47.352 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59035.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85116.log 2024-09-18T01:18:47.352 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.42000.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94808.log 2024-09-18T01:18:47.352 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115476.log.gz 2024-09-18T01:18:47.352 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.85116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138513.log 2024-09-18T01:18:47.353 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94808.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84908.log 2024-09-18T01:18:47.353 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138513.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65498.log 2024-09-18T01:18:47.353 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.84908.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139819.log 2024-09-18T01:18:47.353 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65498.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99118.log 2024-09-18T01:18:47.353 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139819.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36318.log 2024-09-18T01:18:47.353 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99118.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96479.log 2024-09-18T01:18:47.353 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36318.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121782.log 2024-09-18T01:18:47.353 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.116932.log.gz 2024-09-18T01:18:47.353 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.96479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28559.log 2024-09-18T01:18:47.353 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121782.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44957.log 2024-09-18T01:18:47.353 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28559.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.118176.log.gz 2024-09-18T01:18:47.353 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63381.log 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44957.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42705.log 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.63381.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62114.log 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.42705.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102785.log 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.62114.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44514.log 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102785.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66885.log 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44514.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60161.log 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66885.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81790.log 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.60161.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128395.log 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81790.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119429.log 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.128395.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71680.log 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123102.log.gz 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.123093.log.gz 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.119429.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89380.log.gz 2024-09-18T01:18:47.354 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141220.log 2024-09-18T01:18:47.355 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71680.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126233.log 2024-09-18T01:18:47.355 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.141220.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91486.log 2024-09-18T01:18:47.355 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126233.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32361.log 2024-09-18T01:18:47.355 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91486.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102447.log 2024-09-18T01:18:47.355 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32361.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.130010.log.gz 2024-09-18T01:18:47.355 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73018.log 2024-09-18T01:18:47.355 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.141220.log.gz 2024-09-18T01:18:47.355 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102447.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119201.log 2024-09-18T01:18:47.355 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73018.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.129215.log 2024-09-18T01:18:47.355 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.119201.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123081.log 2024-09-18T01:18:47.355 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.129215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38674.log 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123081.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115475.log 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.38674.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54764.log 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129280.log 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54764.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135132.log.gz 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135136.log 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129280.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.107820.log 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135136.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71804.log 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.107820.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135302.log 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71804.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133489.log.gz 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36843.log 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135302.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28660.log 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36843.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120955.log 2024-09-18T01:18:47.360 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28660.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28660.log.gz 2024-09-18T01:18:47.361 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137245.log 2024-09-18T01:18:47.361 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120955.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44580.log 2024-09-18T01:18:47.361 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137245.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139285.log 2024-09-18T01:18:47.361 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44580.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112733.log 2024-09-18T01:18:47.361 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139285.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134921.log.gz 2024-09-18T01:18:47.361 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51870.log 2024-09-18T01:18:47.361 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112733.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91403.log 2024-09-18T01:18:47.363 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51870.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85820.log 2024-09-18T01:18:47.363 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91403.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86736.log 2024-09-18T01:18:47.363 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85820.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119766.log 2024-09-18T01:18:47.363 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.86736.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31446.log 2024-09-18T01:18:47.363 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136741.log.gz 2024-09-18T01:18:47.363 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.119766.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63927.log 2024-09-18T01:18:47.363 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31446.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80112.log 2024-09-18T01:18:47.363 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.63927.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29536.log 2024-09-18T01:18:47.365 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80112.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97524.log 2024-09-18T01:18:47.365 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.29536.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29536.log.gz 2024-09-18T01:18:47.366 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139498.log 2024-09-18T01:18:47.366 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.97524.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.97524.log.gz 2024-09-18T01:18:47.366 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70756.log 2024-09-18T01:18:47.366 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134713.log.gz 2024-09-18T01:18:47.366 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139498.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.86516.log.gz 2024-09-18T01:18:47.366 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51958.log 2024-09-18T01:18:47.367 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70756.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62609.log 2024-09-18T01:18:47.367 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.51958.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85364.log 2024-09-18T01:18:47.367 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.62609.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134077.log 2024-09-18T01:18:47.367 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85364.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123209.log 2024-09-18T01:18:47.368 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134077.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72087.log 2024-09-18T01:18:47.368 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123209.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110740.log 2024-09-18T01:18:47.370 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.72087.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48834.log 2024-09-18T01:18:47.370 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110740.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134503.log 2024-09-18T01:18:47.370 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.48834.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48834.log.gz 2024-09-18T01:18:47.370 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61354.log 2024-09-18T01:18:47.370 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134503.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99246.log 2024-09-18T01:18:47.371 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.61354.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92874.log 2024-09-18T01:18:47.371 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99246.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45673.log 2024-09-18T01:18:47.372 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92874.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94993.log 2024-09-18T01:18:47.372 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45673.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87508.log 2024-09-18T01:18:47.373 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94993.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57349.log 2024-09-18T01:18:47.373 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87508.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72721.log 2024-09-18T01:18:47.374 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57349.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119809.log 2024-09-18T01:18:47.374 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.72721.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116591.log 2024-09-18T01:18:47.374 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119809.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62102.log 2024-09-18T01:18:47.375 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116591.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123711.log 2024-09-18T01:18:47.375 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.62102.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82654.log 2024-09-18T01:18:47.376 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123711.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137620.log 2024-09-18T01:18:47.376 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.82654.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124404.log 2024-09-18T01:18:47.377 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137620.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118367.log 2024-09-18T01:18:47.377 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124404.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56973.log 2024-09-18T01:18:47.378 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.118367.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127940.log 2024-09-18T01:18:47.378 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56973.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84121.log 2024-09-18T01:18:47.379 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.127940.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86573.log 2024-09-18T01:18:47.379 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.84121.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135983.log.gz 2024-09-18T01:18:47.379 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139726.log 2024-09-18T01:18:47.379 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.86573.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89864.log 2024-09-18T01:18:47.381 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139726.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46617.log 2024-09-18T01:18:47.381 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89864.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135091.log 2024-09-18T01:18:47.381 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46617.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55058.log 2024-09-18T01:18:47.382 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135091.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134670.log 2024-09-18T01:18:47.382 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55058.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115540.log 2024-09-18T01:18:47.383 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134670.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119155.log 2024-09-18T01:18:47.383 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115540.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117737.log 2024-09-18T01:18:47.383 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.119155.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51206.log 2024-09-18T01:18:47.383 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117737.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29121.log 2024-09-18T01:18:47.384 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51206.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80656.log 2024-09-18T01:18:47.385 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.29121.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29121.log.gz 2024-09-18T01:18:47.385 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115011.log 2024-09-18T01:18:47.385 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80656.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142466.log 2024-09-18T01:18:47.385 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115011.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33418.log 2024-09-18T01:18:47.386 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.142466.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142466.log.gz 2024-09-18T01:18:47.386 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68079.log 2024-09-18T01:18:47.387 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33418.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43134.log 2024-09-18T01:18:47.387 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68079.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48935.log 2024-09-18T01:18:47.387 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43134.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82680.log 2024-09-18T01:18:47.388 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.48935.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48935.log.gz 2024-09-18T01:18:47.388 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90979.log 2024-09-18T01:18:47.388 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131800.log.gz 2024-09-18T01:18:47.389 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82680.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52267.log 2024-09-18T01:18:47.389 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90979.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102532.log 2024-09-18T01:18:47.390 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52267.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96681.log 2024-09-18T01:18:47.390 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102532.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106761.log 2024-09-18T01:18:47.391 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.96681.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84961.log 2024-09-18T01:18:47.391 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106761.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121221.log 2024-09-18T01:18:47.392 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.84961.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97277.log 2024-09-18T01:18:47.392 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122663.log.gz 2024-09-18T01:18:47.392 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121221.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63199.log 2024-09-18T01:18:47.393 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.97277.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134287.log 2024-09-18T01:18:47.393 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.63199.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80783.log 2024-09-18T01:18:47.393 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134287.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92662.log 2024-09-18T01:18:47.394 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80783.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130104.log 2024-09-18T01:18:47.394 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59941.log 2024-09-18T01:18:47.395 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.130104.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86877.log 2024-09-18T01:18:47.395 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59941.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68567.log 2024-09-18T01:18:47.396 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86877.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116202.log 2024-09-18T01:18:47.396 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68567.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84078.log 2024-09-18T01:18:47.396 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116202.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.116202.log.gz 2024-09-18T01:18:47.397 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49940.log 2024-09-18T01:18:47.397 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84078.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41272.log 2024-09-18T01:18:47.398 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125185.log.gz 2024-09-18T01:18:47.399 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.49940.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118402.log 2024-09-18T01:18:47.399 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41272.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41272.log.gz 2024-09-18T01:18:47.399 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85619.log 2024-09-18T01:18:47.399 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118402.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98329.log 2024-09-18T01:18:47.399 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85619.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137791.log 2024-09-18T01:18:47.400 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98329.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104259.log 2024-09-18T01:18:47.400 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137791.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56719.log 2024-09-18T01:18:47.401 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104259.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101267.log 2024-09-18T01:18:47.402 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56719.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.107451.log 2024-09-18T01:18:47.402 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101267.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125309.log 2024-09-18T01:18:47.403 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.107451.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28891.log 2024-09-18T01:18:47.403 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125309.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102491.log 2024-09-18T01:18:47.403 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28891.log: 56.0% -- replaced with /var/log/ceph/cluster2-client.mirror.29942.log.gz 2024-09-18T01:18:47.403 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28891.log.gz 2024-09-18T01:18:47.403 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142017.log 2024-09-18T01:18:47.404 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102491.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97023.log 2024-09-18T01:18:47.404 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.142017.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142017.log.gz 2024-09-18T01:18:47.404 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118290.log 2024-09-18T01:18:47.405 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.97023.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30304.log 2024-09-18T01:18:47.405 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118290.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140067.log 2024-09-18T01:18:47.406 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.30304.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.97615.log.gz 2024-09-18T01:18:47.406 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85136.log 2024-09-18T01:18:47.407 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140067.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58406.log 2024-09-18T01:18:47.407 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85136.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108536.log 2024-09-18T01:18:47.408 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58406.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57475.log 2024-09-18T01:18:47.408 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.108536.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89843.log.gz 2024-09-18T01:18:47.408 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119697.log 2024-09-18T01:18:47.408 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.98890.log.gz 2024-09-18T01:18:47.408 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39421.log 2024-09-18T01:18:47.409 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119697.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124846.log 2024-09-18T01:18:47.410 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39421.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135558.log 2024-09-18T01:18:47.410 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124846.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138638.log 2024-09-18T01:18:47.411 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135558.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82560.log 2024-09-18T01:18:47.411 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77288.log.gz 2024-09-18T01:18:47.411 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138638.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141953.log 2024-09-18T01:18:47.411 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82560.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88393.log 2024-09-18T01:18:47.412 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141953.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132884.log 2024-09-18T01:18:47.412 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.85116.log.gz 2024-09-18T01:18:47.413 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88393.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80741.log.gz 2024-09-18T01:18:47.413 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71258.log 2024-09-18T01:18:47.413 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132884.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40049.log 2024-09-18T01:18:47.414 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71258.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.71258.log.gz 2024-09-18T01:18:47.414 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58241.log 2024-09-18T01:18:47.414 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.40049.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.40049.log.gz 2024-09-18T01:18:47.415 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30236.log 2024-09-18T01:18:47.415 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82233.log.gz 2024-09-18T01:18:47.415 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58241.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49070.log 2024-09-18T01:18:47.417 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.30236.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97662.log 2024-09-18T01:18:47.417 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.49070.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109109.log 2024-09-18T01:18:47.417 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.141953.log.gz 2024-09-18T01:18:47.417 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.97662.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.97662.log.gz 2024-09-18T01:18:47.417 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73557.log 2024-09-18T01:18:47.418 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.109109.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.109109.log.gz 2024-09-18T01:18:47.418 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114633.log 2024-09-18T01:18:47.419 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73557.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96251.log 2024-09-18T01:18:47.419 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114633.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42634.log.gz 2024-09-18T01:18:47.419 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54615.log 2024-09-18T01:18:47.419 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.96251.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97110.log 2024-09-18T01:18:47.420 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54615.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74820.log 2024-09-18T01:18:47.421 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.97110.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.93805.log 2024-09-18T01:18:47.421 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74820.log: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.39408.log.gz 2024-09-18T01:18:47.421 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52400.log 2024-09-18T01:18:47.421 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.93805.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65108.log 2024-09-18T01:18:47.422 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52400.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110588.log 2024-09-18T01:18:47.423 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65108.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47485.log 2024-09-18T01:18:47.423 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110588.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130352.log 2024-09-18T01:18:47.423 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47485.log: gzip -5 89.9% --verbose -- -- replaced with /var/log/ceph/cluster1-client.mirror.89022.log.gz 2024-09-18T01:18:47.424 INFO:teuthology.orchestra.run.smithi049.stderr: /var/log/ceph/cluster2-client.admin.61603.log 2024-09-18T01:18:47.424 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130352.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85592.log 2024-09-18T01:18:47.425 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61603.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61038.log 2024-09-18T01:18:47.425 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85592.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98933.log 2024-09-18T01:18:47.426 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61038.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37879.log.gz 2024-09-18T01:18:47.426 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124555.log 2024-09-18T01:18:47.426 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98933.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112693.log 2024-09-18T01:18:47.427 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124555.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48159.log 2024-09-18T01:18:47.427 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112693.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50795.log 2024-09-18T01:18:47.428 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48159.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57900.log 2024-09-18T01:18:47.428 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50795.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83580.log 2024-09-18T01:18:47.428 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57900.log: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.128707.log.gz 2024-09-18T01:18:47.429 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70755.log 2024-09-18T01:18:47.430 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.83580.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131191.log 2024-09-18T01:18:47.430 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70755.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85665.log 2024-09-18T01:18:47.430 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131191.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109707.log 2024-09-18T01:18:47.431 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.85665.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108229.log 2024-09-18T01:18:47.431 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.109707.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109265.log 2024-09-18T01:18:47.431 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.108229.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76265.log 2024-09-18T01:18:47.432 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.109265.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31797.log 2024-09-18T01:18:47.433 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76265.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33628.log.gz 2024-09-18T01:18:47.433 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114485.log 2024-09-18T01:18:47.434 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31797.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113642.log 2024-09-18T01:18:47.434 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114485.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120057.log 2024-09-18T01:18:47.434 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113642.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125942.log 2024-09-18T01:18:47.434 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34425.log.gz 2024-09-18T01:18:47.434 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120057.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79645.log 2024-09-18T01:18:47.435 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125942.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76947.log 2024-09-18T01:18:47.435 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79645.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116261.log 2024-09-18T01:18:47.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76947.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103673.log 2024-09-18T01:18:47.436 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116261.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65205.log 2024-09-18T01:18:47.437 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103673.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52866.log 2024-09-18T01:18:47.437 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.65205.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.65205.log.gz 2024-09-18T01:18:47.438 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32317.log 2024-09-18T01:18:47.438 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52866.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.94078.log 2024-09-18T01:18:47.439 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32317.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81625.log 2024-09-18T01:18:47.439 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.94078.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.94078.log.gz 2024-09-18T01:18:47.439 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53707.log 2024-09-18T01:18:47.440 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81625.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52759.log 2024-09-18T01:18:47.441 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53707.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105036.log 2024-09-18T01:18:47.441 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75696.log 2024-09-18T01:18:47.441 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105036.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43531.log 2024-09-18T01:18:47.442 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75696.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70756.log.gz 2024-09-18T01:18:47.442 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68228.log 2024-09-18T01:18:47.442 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43531.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79479.log 2024-09-18T01:18:47.443 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68228.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35307.log 2024-09-18T01:18:47.443 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142234.log 2024-09-18T01:18:47.444 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35307.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109523.log 2024-09-18T01:18:47.444 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.142234.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142234.log.gz 2024-09-18T01:18:47.444 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120557.log 2024-09-18T01:18:47.445 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.109523.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28579.log 2024-09-18T01:18:47.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120557.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36043.log 2024-09-18T01:18:47.446 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28579.log: 0.0% 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.28579.log.gz 2024-09-18T01:18:47.446 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.106744.log.gz 2024-09-18T01:18:47.446 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124556.log 2024-09-18T01:18:47.447 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36043.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132884.log.gz 2024-09-18T01:18:47.447 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113222.log 2024-09-18T01:18:47.447 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124556.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129624.log 2024-09-18T01:18:47.448 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113222.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90473.log 2024-09-18T01:18:47.448 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129624.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129624.log.gz 2024-09-18T01:18:47.448 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61276.log 2024-09-18T01:18:47.449 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90473.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.140992.log 2024-09-18T01:18:47.449 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61276.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68229.log 2024-09-18T01:18:47.450 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.140992.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130063.log 2024-09-18T01:18:47.450 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68229.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134716.log 2024-09-18T01:18:47.451 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.130063.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134927.log 2024-09-18T01:18:47.451 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134716.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.140992.log.gz 2024-09-18T01:18:47.451 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122512.log 2024-09-18T01:18:47.452 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134927.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58111.log 2024-09-18T01:18:47.452 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122512.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31306.log 2024-09-18T01:18:47.453 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58111.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43418.log 2024-09-18T01:18:47.453 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.31306.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31306.log.gz 2024-09-18T01:18:47.454 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90497.log 2024-09-18T01:18:47.454 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43418.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29584.log 2024-09-18T01:18:47.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109151.log 2024-09-18T01:18:47.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.29584.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97978.log 2024-09-18T01:18:47.455 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109151.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41837.log 2024-09-18T01:18:47.456 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.97978.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59834.log.gz 2024-09-18T01:18:47.456 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101730.log 2024-09-18T01:18:47.456 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41837.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120885.log 2024-09-18T01:18:47.457 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101730.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.110112.log 2024-09-18T01:18:47.457 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120885.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63293.log.gz 2024-09-18T01:18:47.458 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34475.log 2024-09-18T01:18:47.458 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.110112.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36572.log 2024-09-18T01:18:47.458 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34475.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100554.log 2024-09-18T01:18:47.459 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36572.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101184.log 2024-09-18T01:18:47.460 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100554.log: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.28559.log.gz 2024-09-18T01:18:47.460 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53485.log 2024-09-18T01:18:47.460 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101184.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50847.log.gz 2024-09-18T01:18:47.461 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99903.log 2024-09-18T01:18:47.461 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53485.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101099.log 2024-09-18T01:18:47.461 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99903.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76730.log 2024-09-18T01:18:47.462 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101099.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96292.log 2024-09-18T01:18:47.462 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76730.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95954.log 2024-09-18T01:18:47.462 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.96292.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86327.log 2024-09-18T01:18:47.463 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95954.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121656.log 2024-09-18T01:18:47.463 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86327.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95971.log 2024-09-18T01:18:47.464 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121656.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39791.log 2024-09-18T01:18:47.464 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95971.log: gzip 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66715.log.gz -5 2024-09-18T01:18:47.464 INFO:teuthology.orchestra.run.smithi049.stderr: --verbose -- /var/log/ceph/cluster2-client.admin.84683.log 2024-09-18T01:18:47.465 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39791.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57771.log 2024-09-18T01:18:47.465 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84683.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113431.log 2024-09-18T01:18:47.466 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57771.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93295.log 2024-09-18T01:18:47.466 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113431.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28631.log 2024-09-18T01:18:47.467 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93295.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46640.log 2024-09-18T01:18:47.467 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28631.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28631.log.gz 2024-09-18T01:18:47.467 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41214.log 2024-09-18T01:18:47.468 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96935.log 2024-09-18T01:18:47.468 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41214.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45884.log 2024-09-18T01:18:47.469 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.96935.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66758.log 2024-09-18T01:18:47.470 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45884.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98001.log 2024-09-18T01:18:47.470 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66758.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137031.log 2024-09-18T01:18:47.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.98001.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133330.log 2024-09-18T01:18:47.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137031.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132982.log 2024-09-18T01:18:47.471 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133330.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120025.log 2024-09-18T01:18:47.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132982.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30531.log 2024-09-18T01:18:47.472 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120025.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36002.log 2024-09-18T01:18:47.473 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.30531.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30531.log.gz 2024-09-18T01:18:47.473 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102194.log 2024-09-18T01:18:47.474 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36002.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139449.log 2024-09-18T01:18:47.474 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102194.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53306.log 2024-09-18T01:18:47.474 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139449.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78965.log 2024-09-18T01:18:47.475 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53306.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91613.log 2024-09-18T01:18:47.475 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78965.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57452.log 2024-09-18T01:18:47.476 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91613.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78504.log 2024-09-18T01:18:47.477 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57452.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46935.log 2024-09-18T01:18:47.477 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78504.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73404.log 2024-09-18T01:18:47.478 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46935.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106215.log 2024-09-18T01:18:47.478 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73404.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130379.log 2024-09-18T01:18:47.478 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32361.log.gz 2024-09-18T01:18:47.478 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.106215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41417.log 2024-09-18T01:18:47.479 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130379.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118537.log 2024-09-18T01:18:47.480 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69203.log 2024-09-18T01:18:47.480 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118537.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118537.log.gz 2024-09-18T01:18:47.480 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102915.log 2024-09-18T01:18:47.481 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69203.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56718.log 2024-09-18T01:18:47.481 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36043.log.gz 2024-09-18T01:18:47.481 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102915.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106176.log 2024-09-18T01:18:47.481 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56718.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45840.log 2024-09-18T01:18:47.482 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60721.log 2024-09-18T01:18:47.483 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45840.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108363.log 2024-09-18T01:18:47.484 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.60721.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141722.log 2024-09-18T01:18:47.484 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.85592.log.gz 2024-09-18T01:18:47.484 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.83017.log.gz 2024-09-18T01:18:47.484 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.108363.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77131.log 2024-09-18T01:18:47.484 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141722.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74418.log 2024-09-18T01:18:47.485 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77131.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68442.log 2024-09-18T01:18:47.486 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.74418.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44117.log 2024-09-18T01:18:47.486 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114276.log 2024-09-18T01:18:47.486 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44117.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46473.log 2024-09-18T01:18:47.487 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114276.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74365.log 2024-09-18T01:18:47.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92874.log.gz 2024-09-18T01:18:47.487 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46473.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115180.log 2024-09-18T01:18:47.488 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.74365.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114698.log 2024-09-18T01:18:47.489 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115180.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105869.log 2024-09-18T01:18:47.489 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114698.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131111.log 2024-09-18T01:18:47.490 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.105869.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76415.log 2024-09-18T01:18:47.490 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.141722.log.gz 2024-09-18T01:18:47.490 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131111.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118553.log 2024-09-18T01:18:47.491 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76415.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132559.log 2024-09-18T01:18:47.491 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118553.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118553.log.gz 2024-09-18T01:18:47.491 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4%gzip -- replaced with /var/log/ceph/cluster2-client.admin.84078.log.gz -5 2024-09-18T01:18:47.491 INFO:teuthology.orchestra.run.smithi049.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.104385.log 2024-09-18T01:18:47.492 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132559.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93293.log 2024-09-18T01:18:47.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104385.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48326.log 2024-09-18T01:18:47.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93293.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64994.log 2024-09-18T01:18:47.493 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48326.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102916.log 2024-09-18T01:18:47.494 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.64994.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84961.log.gz 2024-09-18T01:18:47.494 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66189.log 2024-09-18T01:18:47.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102916.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.99007.log 2024-09-18T01:18:47.495 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66189.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102831.log 2024-09-18T01:18:47.496 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.99007.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127722.log 2024-09-18T01:18:47.496 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102831.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129263.log 2024-09-18T01:18:47.497 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127722.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140094.log 2024-09-18T01:18:47.497 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129263.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129263.log.gz 2024-09-18T01:18:47.497 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67472.log 2024-09-18T01:18:47.498 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140094.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100218.log 2024-09-18T01:18:47.498 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67472.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67029.log 2024-09-18T01:18:47.499 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100218.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63179.log 2024-09-18T01:18:47.499 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67029.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85686.log 2024-09-18T01:18:47.500 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.63179.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101522.log.gz 2024-09-18T01:18:47.500 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51475.log 2024-09-18T01:18:47.500 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101563.log.gz 2024-09-18T01:18:47.501 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.85686.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64752.log 2024-09-18T01:18:47.501 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.51475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95956.log 2024-09-18T01:18:47.501 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.64752.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.64752.log.gz 2024-09-18T01:18:47.502 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62013.log 2024-09-18T01:18:47.502 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95956.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101688.log.gz 2024-09-18T01:18:47.503 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121337.log 2024-09-18T01:18:47.503 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.62013.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49632.log 2024-09-18T01:18:47.503 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121337.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90982.log 2024-09-18T01:18:47.504 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.49632.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101897.log 2024-09-18T01:18:47.504 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90982.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99587.log 2024-09-18T01:18:47.505 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101897.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111600.log.gz 2024-09-18T01:18:47.505 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105785.log 2024-09-18T01:18:47.505 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99587.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85779.log 2024-09-18T01:18:47.506 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.105785.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139667.log 2024-09-18T01:18:47.506 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.85779.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125605.log 2024-09-18T01:18:47.507 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.139667.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141907.log 2024-09-18T01:18:47.508 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125605.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87671.log 2024-09-18T01:18:47.508 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.141907.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52891.log 2024-09-18T01:18:47.509 INFO:teuthology.orchestra.run.smithi049.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.141907.log.gz 2024-09-18T01:18:47.509 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87671.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61820.log 2024-09-18T01:18:47.509 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.110002.log.gz 2024-09-18T01:18:47.509 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52891.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110059.log.gz 2024-09-18T01:18:47.509 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67239.log 2024-09-18T01:18:47.510 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.61820.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97297.log 2024-09-18T01:18:47.510 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67239.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108803.log 2024-09-18T01:18:47.511 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.97297.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103594.log 2024-09-18T01:18:47.511 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.108803.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120152.log 2024-09-18T01:18:47.512 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103594.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39685.log 2024-09-18T01:18:47.512 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120152.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44580.log.gz 2024-09-18T01:18:47.513 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102066.log 2024-09-18T01:18:47.513 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39685.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.39685.log.gz 2024-09-18T01:18:47.513 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56402.log 2024-09-18T01:18:47.514 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102066.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39388.log 2024-09-18T01:18:47.514 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56402.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30716.log 2024-09-18T01:18:47.515 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.39388.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71424.log 2024-09-18T01:18:47.515 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.30716.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48199.log.gz 2024-09-18T01:18:47.516 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48198.log.gz 2024-09-18T01:18:47.516 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93166.log 2024-09-18T01:18:47.516 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.71424.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34949.log 2024-09-18T01:18:47.516 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.75696.log.gz 2024-09-18T01:18:47.516 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93166.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88539.log 2024-09-18T01:18:47.517 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34949.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82044.log 2024-09-18T01:18:47.522 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88539.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114487.log.gz 2024-09-18T01:18:47.522 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122143.log 2024-09-18T01:18:47.522 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.82044.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139889.log 2024-09-18T01:18:47.522 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122143.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124131.log 2024-09-18T01:18:47.522 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139889.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54931.log 2024-09-18T01:18:47.522 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124131.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102193.log 2024-09-18T01:18:47.522 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54931.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55838.log.gz 2024-09-18T01:18:47.522 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89505.log 2024-09-18T01:18:47.523 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102193.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66716.log 2024-09-18T01:18:47.523 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89505.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69455.log 2024-09-18T01:18:47.523 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66716.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49218.log 2024-09-18T01:18:47.523 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69455.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63467.log 2024-09-18T01:18:47.524 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.49218.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63126.log 2024-09-18T01:18:47.524 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.63467.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90284.log 2024-09-18T01:18:47.524 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.63126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86620.log 2024-09-18T01:18:47.524 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90284.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100805.log 2024-09-18T01:18:47.524 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86620.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120820.log 2024-09-18T01:18:47.525 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100805.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53895.log 2024-09-18T01:18:47.526 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120820.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37818.log 2024-09-18T01:18:47.526 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.53895.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91554.log 2024-09-18T01:18:47.527 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.37818.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37818.log.gz 2024-09-18T01:18:47.527 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134501.log 2024-09-18T01:18:47.527 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91554.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31962.log 2024-09-18T01:18:47.528 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134501.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37814.log 2024-09-18T01:18:47.528 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31962.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135772.log 2024-09-18T01:18:47.529 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.37814.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37814.log.gz 2024-09-18T01:18:47.529 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76088.log 2024-09-18T01:18:47.529 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124846.log.gz 2024-09-18T01:18:47.529 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135772.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77924.log 2024-09-18T01:18:47.530 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.76088.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45041.log 2024-09-18T01:18:47.531 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77924.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88602.log 2024-09-18T01:18:47.531 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45041.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45800.log 2024-09-18T01:18:47.531 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88602.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.127940.log.gz 2024-09-18T01:18:47.531 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.127526.log.gz 2024-09-18T01:18:47.531 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113283.log 2024-09-18T01:18:47.532 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45800.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84300.log 2024-09-18T01:18:47.532 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113283.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.114276.log.gz 2024-09-18T01:18:47.533 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106443.log 2024-09-18T01:18:47.533 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84300.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28735.log 2024-09-18T01:18:47.533 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106443.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118519.log 2024-09-18T01:18:47.534 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28735.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28735.log.gz 2024-09-18T01:18:47.534 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109360.log 2024-09-18T01:18:47.534 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118519.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118519.log.gz 2024-09-18T01:18:47.535 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118532.log 2024-09-18T01:18:47.535 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109360.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.128395.log.gz 2024-09-18T01:18:47.535 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63220.log 2024-09-18T01:18:47.536 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118532.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118532.log.gz 2024-09-18T01:18:47.536 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74880.log 2024-09-18T01:18:47.537 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.63220.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60578.log 2024-09-18T01:18:47.537 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.74880.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81624.log 2024-09-18T01:18:47.538 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60578.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61089.log 2024-09-18T01:18:47.538 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81624.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126946.log 2024-09-18T01:18:47.539 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61089.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60279.log 2024-09-18T01:18:47.539 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126946.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127423.log 2024-09-18T01:18:47.540 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.60279.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98954.log 2024-09-18T01:18:47.540 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.127423.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.127423.log.gz 2024-09-18T01:18:47.540 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46406.log 2024-09-18T01:18:47.541 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98954.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92330.log 2024-09-18T01:18:47.541 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46406.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131842.log 2024-09-18T01:18:47.542 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92330.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111958.log 2024-09-18T01:18:47.542 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.131842.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.131842.log.gz 2024-09-18T01:18:47.542 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110024.log 2024-09-18T01:18:47.543 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111958.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90918.log 2024-09-18T01:18:47.543 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110024.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124218.log 2024-09-18T01:18:47.544 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90918.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72844.log 2024-09-18T01:18:47.544 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124218.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96426.log 2024-09-18T01:18:47.545 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.72844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123796.log 2024-09-18T01:18:47.545 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.96426.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82960.log 2024-09-18T01:18:47.546 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123796.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109216.log 2024-09-18T01:18:47.546 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.82960.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66233.log 2024-09-18T01:18:47.546 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109216.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105054.log 2024-09-18T01:18:47.547 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66233.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45378.log 2024-09-18T01:18:47.547 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.105054.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.105054.log.gz 2024-09-18T01:18:47.547 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114382.log 2024-09-18T01:18:47.548 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45378.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125058.log 2024-09-18T01:18:47.549 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114382.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109977.log 2024-09-18T01:18:47.549 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125058.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55478.log 2024-09-18T01:18:47.550 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109977.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85706.log 2024-09-18T01:18:47.550 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55478.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41406.log 2024-09-18T01:18:47.551 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85706.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124319.log 2024-09-18T01:18:47.551 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41406.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41406.log.gz 2024-09-18T01:18:47.551 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50056.log 2024-09-18T01:18:47.552 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124319.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87306.log 2024-09-18T01:18:47.552 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50056.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57664.log 2024-09-18T01:18:47.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87306.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135982.log 2024-09-18T01:18:47.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57664.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125521.log 2024-09-18T01:18:47.554 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135982.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77131.log.gz 2024-09-18T01:18:47.554 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68357.log 2024-09-18T01:18:47.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125521.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.132211.log 2024-09-18T01:18:47.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68357.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68399.log 2024-09-18T01:18:47.555 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.132211.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.132211.log.gz 2024-09-18T01:18:47.556 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33940.log 2024-09-18T01:18:47.557 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67029.log.gz 2024-09-18T01:18:47.557 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68399.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38255.log 2024-09-18T01:18:47.557 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33940.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90138.log 2024-09-18T01:18:47.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.38255.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66157.log 2024-09-18T01:18:47.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90138.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86774.log 2024-09-18T01:18:47.558 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66157.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126026.log 2024-09-18T01:18:47.559 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86774.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118994.log 2024-09-18T01:18:47.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126026.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107659.log 2024-09-18T01:18:47.560 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.118994.log: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.98001.log.gz 2024-09-18T01:18:47.561 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.97978.log.gz 2024-09-18T01:18:47.561 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91970.log 2024-09-18T01:18:47.561 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.107659.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.97933.log.gz 2024-09-18T01:18:47.562 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117722.log 2024-09-18T01:18:47.562 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91970.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45252.log 2024-09-18T01:18:47.562 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117722.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119181.log 2024-09-18T01:18:47.563 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45252.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137792.log 2024-09-18T01:18:47.563 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.119181.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80614.log 2024-09-18T01:18:47.563 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137792.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64246.log 2024-09-18T01:18:47.564 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80614.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32175.log 2024-09-18T01:18:47.565 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64246.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.109151.log.gz 2024-09-18T01:18:47.565 INFO:teuthology.orchestra.run.smithi049.stderr: 20.5% -- replaced with /var/log/ceph/cluster1-client.admin.109265.log.gz 2024-09-18T01:18:47.565 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117974.log 2024-09-18T01:18:47.565 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32175.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107020.log 2024-09-18T01:18:47.566 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.117974.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96190.log 2024-09-18T01:18:47.567 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107020.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83968.log 2024-09-18T01:18:47.567 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.96190.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96190.log.gz 2024-09-18T01:18:47.567 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36422.log 2024-09-18T01:18:47.567 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83968.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.102491.log.gz 2024-09-18T01:18:47.568 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87201.log 2024-09-18T01:18:47.568 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36422.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108493.log 2024-09-18T01:18:47.569 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87201.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127651.log 2024-09-18T01:18:47.569 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.108493.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84557.log 2024-09-18T01:18:47.569 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127651.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58365.log 2024-09-18T01:18:47.570 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36318.log.gz 2024-09-18T01:18:47.570 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84557.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84557.log.gz 2024-09-18T01:18:47.570 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126991.log 2024-09-18T01:18:47.571 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58365.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.132439.log 2024-09-18T01:18:47.571 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134499.log 2024-09-18T01:18:47.572 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.132439.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120423.log 2024-09-18T01:18:47.572 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134499.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141307.log 2024-09-18T01:18:47.573 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120423.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36989.log 2024-09-18T01:18:47.574 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.141307.log: gzip -5 87.4% --verbose -- -- replaced with /var/log/ceph/cluster1-client.admin.141307.log.gz 2024-09-18T01:18:47.574 INFO:teuthology.orchestra.run.smithi049.stderr: /var/log/ceph/cluster2-client.admin.140687.log 2024-09-18T01:18:47.574 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88393.log.gz 2024-09-18T01:18:47.574 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36989.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89593.log 2024-09-18T01:18:47.575 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140687.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141880.log 2024-09-18T01:18:47.575 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89593.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131489.log 2024-09-18T01:18:47.576 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.141880.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.72258.log.gz 2024-09-18T01:18:47.576 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.141880.log.gz 2024-09-18T01:18:47.577 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.129896.log 2024-09-18T01:18:47.577 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.131489.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.131489.log.gz 2024-09-18T01:18:47.577 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137457.log 2024-09-18T01:18:47.578 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.129896.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115052.log 2024-09-18T01:18:47.578 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137457.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95845.log 2024-09-18T01:18:47.578 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115052.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75976.log 2024-09-18T01:18:47.579 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95845.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80700.log.gz 2024-09-18T01:18:47.579 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45312.log 2024-09-18T01:18:47.579 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75976.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86393.log 2024-09-18T01:18:47.580 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45312.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114066.log 2024-09-18T01:18:47.580 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86393.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128169.log 2024-09-18T01:18:47.581 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114066.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133910.log 2024-09-18T01:18:47.581 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.128169.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56508.log 2024-09-18T01:18:47.582 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133910.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78044.log 2024-09-18T01:18:47.583 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56508.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36361.log 2024-09-18T01:18:47.583 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78044.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59036.log 2024-09-18T01:18:47.583 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36361.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47271.log 2024-09-18T01:18:47.583 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48326.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.48325.log.gz 2024-09-18T01:18:47.583 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:47.584 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59036.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69346.log 2024-09-18T01:18:47.584 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47271.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86215.log 2024-09-18T01:18:47.585 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69346.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96976.log 2024-09-18T01:18:47.585 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74932.log 2024-09-18T01:18:47.586 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.96976.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75241.log 2024-09-18T01:18:47.586 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74932.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102363.log 2024-09-18T01:18:47.587 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.75241.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50740.log 2024-09-18T01:18:47.587 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102363.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115222.log 2024-09-18T01:18:47.588 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50740.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41539.log 2024-09-18T01:18:47.588 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115222.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92392.log 2024-09-18T01:18:47.589 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.41539.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55393.log 2024-09-18T01:18:47.590 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92392.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135772.log.gz 2024-09-18T01:18:47.590 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111474.log 2024-09-18T01:18:47.590 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55393.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73530.log 2024-09-18T01:18:47.591 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111474.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87400.log 2024-09-18T01:18:47.591 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.73530.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112946.log 2024-09-18T01:18:47.592 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87400.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114485.log.gz 2024-09-18T01:18:47.592 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40241.log 2024-09-18T01:18:47.592 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112946.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48643.log 2024-09-18T01:18:47.593 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.40241.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122871.log 2024-09-18T01:18:47.593 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48643.log: 88.4%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.51186.log.gz -5 2024-09-18T01:18:47.593 INFO:teuthology.orchestra.run.smithi049.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.122302.log 2024-09-18T01:18:47.594 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122871.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92957.log 2024-09-18T01:18:47.595 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122302.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43089.log 2024-09-18T01:18:47.595 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92957.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132178.log 2024-09-18T01:18:47.596 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43089.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47672.log 2024-09-18T01:18:47.597 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132178.log: 19.4%gzip -- replaced with /var/log/ceph/cluster1-client.admin.64994.log.gz 2024-09-18T01:18:47.597 INFO:teuthology.orchestra.run.smithi049.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79434.log 2024-09-18T01:18:47.597 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47672.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.106589.log.gz 2024-09-18T01:18:47.597 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92790.log 2024-09-18T01:18:47.597 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79434.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118436.log 2024-09-18T01:18:47.598 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92790.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128677.log 2024-09-18T01:18:47.599 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.118436.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90981.log 2024-09-18T01:18:47.599 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.128677.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.128677.log.gz 2024-09-18T01:18:47.599 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53852.log 2024-09-18T01:18:47.600 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90981.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31552.log 2024-09-18T01:18:47.600 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.53852.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136614.log 2024-09-18T01:18:47.601 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.31552.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86837.log 2024-09-18T01:18:47.601 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136614.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.110588.log.gz 2024-09-18T01:18:47.601 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61827.log 2024-09-18T01:18:47.602 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86837.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55563.log 2024-09-18T01:18:47.602 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.61827.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120964.log 2024-09-18T01:18:47.603 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55563.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51002.log 2024-09-18T01:18:47.604 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.120964.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43316.log 2024-09-18T01:18:47.604 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.51002.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.51002.log.gz 2024-09-18T01:18:47.604 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47907.log 2024-09-18T01:18:47.604 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43316.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54724.log 2024-09-18T01:18:47.605 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47907.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123713.log 2024-09-18T01:18:47.606 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54724.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.124131.log.gz 2024-09-18T01:18:47.606 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79693.log 2024-09-18T01:18:47.606 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123713.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73648.log 2024-09-18T01:18:47.606 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79693.log: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.30716.log.gz 2024-09-18T01:18:47.607 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70463.log 2024-09-18T01:18:47.607 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.134503.log.gz 2024-09-18T01:18:47.608 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73648.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73648.log.gz 2024-09-18T01:18:47.608 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141038.log 2024-09-18T01:18:47.608 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.70463.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122789.log 2024-09-18T01:18:47.609 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.141038.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106108.log 2024-09-18T01:18:47.609 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122789.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95960.log 2024-09-18T01:18:47.610 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.106108.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79984.log 2024-09-18T01:18:47.610 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95960.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123589.log.gz 2024-09-18T01:18:47.610 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31264.log 2024-09-18T01:18:47.611 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79984.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42510.log 2024-09-18T01:18:47.611 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31264.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116043.log 2024-09-18T01:18:47.612 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42510.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39386.log 2024-09-18T01:18:47.613 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.116043.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73255.log 2024-09-18T01:18:47.613 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.39386.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52713.log 2024-09-18T01:18:47.613 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.73255.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72997.log 2024-09-18T01:18:47.614 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52713.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110823.log 2024-09-18T01:18:47.614 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.141038.log.gz 2024-09-18T01:18:47.615 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.72997.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72997.log.gz 2024-09-18T01:18:47.615 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77709.log 2024-09-18T01:18:47.615 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111894.log.gz 2024-09-18T01:18:47.615 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110823.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116001.log 2024-09-18T01:18:47.616 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77709.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44412.log 2024-09-18T01:18:47.616 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.116001.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66098.log 2024-09-18T01:18:47.617 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113283.log.gz 2024-09-18T01:18:47.617 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44412.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36569.log 2024-09-18T01:18:47.617 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66098.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108094.log 2024-09-18T01:18:47.619 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36569.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135260.log 2024-09-18T01:18:47.619 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.108094.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30533.log 2024-09-18T01:18:47.619 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135260.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115432.log 2024-09-18T01:18:47.620 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.30533.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74620.log 2024-09-18T01:18:47.620 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115432.log: 87.5%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.120885.log.gz 2024-09-18T01:18:47.620 INFO:teuthology.orchestra.run.smithi049.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.admin.72752.log 2024-09-18T01:18:47.621 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.74620.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37302.log 2024-09-18T01:18:47.621 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.72752.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127306.log 2024-09-18T01:18:47.621 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37302.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32174.log 2024-09-18T01:18:47.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.127306.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88666.log 2024-09-18T01:18:47.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32174.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77347.log 2024-09-18T01:18:47.623 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88666.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83465.log 2024-09-18T01:18:47.623 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77347.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126236.log 2024-09-18T01:18:47.624 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.83465.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66316.log 2024-09-18T01:18:47.624 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126236.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116350.log 2024-09-18T01:18:47.625 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66316.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35306.log 2024-09-18T01:18:47.625 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.116350.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39388.log.gz 2024-09-18T01:18:47.625 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69704.log 2024-09-18T01:18:47.626 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35306.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113321.log 2024-09-18T01:18:47.627 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69704.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52139.log 2024-09-18T01:18:47.627 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113321.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35096.log 2024-09-18T01:18:47.627 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.52139.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59162.log 2024-09-18T01:18:47.629 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.35096.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124824.log.gz 2024-09-18T01:18:47.629 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126526.log 2024-09-18T01:18:47.629 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59162.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133405.log 2024-09-18T01:18:47.630 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.126526.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95257.log 2024-09-18T01:18:47.630 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133405.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121301.log 2024-09-18T01:18:47.630 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.95257.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46364.log 2024-09-18T01:18:47.631 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121301.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107323.log 2024-09-18T01:18:47.631 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46364.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92812.log 2024-09-18T01:18:47.631 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.107323.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107323.log.gz 2024-09-18T01:18:47.631 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52780.log 2024-09-18T01:18:47.632 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.92812.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80615.log 2024-09-18T01:18:47.633 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.52780.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.52780.log.gz 2024-09-18T01:18:47.633 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131337.log 2024-09-18T01:18:47.633 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80615.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54570.log 2024-09-18T01:18:47.634 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.131337.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39850.log 2024-09-18T01:18:47.634 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54570.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134208.log 2024-09-18T01:18:47.635 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.39850.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113223.log 2024-09-18T01:18:47.637 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134208.log: /var/log/ceph/cluster1-client.mirror.113223.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133330.log.gz 2024-09-18T01:18:47.640 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.87671.log.gz 2024-09-18T01:18:47.641 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.86620.log.gz 2024-09-18T01:18:47.641 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.134716.log.gz 2024-09-18T01:18:47.646 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.139449.log.gz 2024-09-18T01:18:47.646 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.82960.log.gz 2024-09-18T01:18:47.652 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115389.log.gz 2024-09-18T01:18:47.660 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100763.log.gz 2024-09-18T01:18:47.664 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46935.log.gz 2024-09-18T01:18:47.667 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.64246.log.gz 2024-09-18T01:18:47.671 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.41214.log.gz 2024-09-18T01:18:47.676 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58241.log.gz 2024-09-18T01:18:47.680 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57475.log.gz 2024-09-18T01:18:47.687 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47907.log.gz 2024-09-18T01:18:47.689 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.32578.log.gz 2024-09-18T01:18:47.690 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.40241.log.gz 2024-09-18T01:18:47.693 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91339.log.gz 2024-09-18T01:18:47.695 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90981.log.gz 2024-09-18T01:18:47.695 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90982.log.gz 2024-09-18T01:18:47.696 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.92392.log.gz 2024-09-18T01:18:47.698 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110823.log.gz 2024-09-18T01:18:47.701 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88602.log.gz 2024-09-18T01:18:47.701 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster2-client.admin.95956.log.gz 2024-09-18T01:18:47.703 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74076.log.gz 2024-09-18T01:18:47.704 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89593.log.gz 2024-09-18T01:18:47.705 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78044.log.gz 2024-09-18T01:18:47.707 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36989.log.gz 2024-09-18T01:18:47.710 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103594.log.gz 2024-09-18T01:18:47.710 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103419.log.gz 2024-09-18T01:18:47.710 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103672.log.gz 2024-09-18T01:18:47.711 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115432.log.gz 2024-09-18T01:18:47.713 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45041.log.gz 2024-09-18T01:18:47.714 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.116933.log.gz 2024-09-18T01:18:47.714 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45168.log.gz 2024-09-18T01:18:47.717 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.52891.log.gz 2024-09-18T01:18:47.721 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.60279.log.gz 2024-09-18T01:18:47.724 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34949.log.gz 2024-09-18T01:18:47.727 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80614.log.gz 2024-09-18T01:18:47.728 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80615.log.gz 2024-09-18T01:18:47.730 INFO:teuthology.orchestra.run.smithi049.stderr: 68.0% -- replaced with /var/log/ceph/cluster1-client.mirror.118994.log.gz 2024-09-18T01:18:47.734 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66316.log.gz 2024-09-18T01:18:47.738 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72742.log.gz 2024-09-18T01:18:47.741 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.120288.log.gz 2024-09-18T01:18:47.747 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.60512.log.gz 2024-09-18T01:18:47.747 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.60721.log.gz 2024-09-18T01:18:47.751 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45800.log.gz 2024-09-18T01:18:47.753 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45840.log.gz 2024-09-18T01:18:47.755 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52139.log.gz 2024-09-18T01:18:47.764 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34573.log.gz 2024-09-18T01:18:47.767 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85189.log.gz 2024-09-18T01:18:47.768 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86289.log.gz 2024-09-18T01:18:47.769 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.37907.log.gz 2024-09-18T01:18:47.772 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89715.log.gz 2024-09-18T01:18:47.774 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88496.log.gz 2024-09-18T01:18:47.778 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59204.log.gz 2024-09-18T01:18:47.778 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.51341.log.gz 2024-09-18T01:18:47.784 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.82849.log.gz 2024-09-18T01:18:47.786 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.98213.log.gz 2024-09-18T01:18:47.789 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76769.log.gz 2024-09-18T01:18:47.789 INFO:teuthology.orchestra.run.smithi049.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73734.log.gz 2024-09-18T01:18:47.793 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77004.log.gz 2024-09-18T01:18:47.796 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69158.log.gz 2024-09-18T01:18:47.799 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69203.log.gz 2024-09-18T01:18:47.801 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.48643.log.gz 2024-09-18T01:18:47.804 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44581.log.gz 2024-09-18T01:18:47.806 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47672.log.gz 2024-09-18T01:18:47.811 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78126.log.gz 2024-09-18T01:18:47.811 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78129.log.gz 2024-09-18T01:18:47.812 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79218.log.gz 2024-09-18T01:18:47.815 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.74820.log.gz 2024-09-18T01:18:47.815 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.76087.log.gz 2024-09-18T01:18:47.816 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48200.log.gz 2024-09-18T01:18:47.819 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115266.log.gz 2024-09-18T01:18:47.823 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.113011.log.gz 2024-09-18T01:18:47.823 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.137872.log.gz 2024-09-18T01:18:47.825 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130747.log.gz 2024-09-18T01:18:47.825 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125161.log.gz 2024-09-18T01:18:47.826 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45523.log.gz 2024-09-18T01:18:47.828 INFO:teuthology.orchestra.run.smithi049.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51528.log.gz 2024-09-18T01:18:47.831 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.139426.log.gz 2024-09-18T01:18:47.834 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134923.log.gz 2024-09-18T01:18:47.835 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.64754.log.gz 2024-09-18T01:18:47.837 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68357.log.gz 2024-09-18T01:18:47.842 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138003.log.gz 2024-09-18T01:18:47.845 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.106108.log.gz 2024-09-18T01:18:47.849 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114002.log.gz 2024-09-18T01:18:47.853 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.39660.log.gz 2024-09-18T01:18:47.853 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.133238.log.gz 2024-09-18T01:18:47.862 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111432.log.gz 2024-09-18T01:18:47.866 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130291.log.gz 2024-09-18T01:18:47.871 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.129096.log.gz 2024-09-18T01:18:47.874 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126236.log.gz 2024-09-18T01:18:47.885 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136614.log.gz 2024-09-18T01:18:47.885 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92875.log.gz 2024-09-18T01:18:47.897 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.115052.log.gz 2024-09-18T01:18:47.902 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.116001.log.gz 2024-09-18T01:18:47.906 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43780.log.gz 2024-09-18T01:18:47.906 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.94515.log.gz 2024-09-18T01:18:47.911 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68993.log.gz 2024-09-18T01:18:47.915 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.38549.log.gz 2024-09-18T01:18:47.916 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.85250.log.gz 2024-09-18T01:18:47.917 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.91529.log.gz 2024-09-18T01:18:47.917 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.64245.log.gz 2024-09-18T01:18:47.919 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.72290.log.gz 2024-09-18T01:18:47.919 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60812.log.gz 2024-09-18T01:18:47.922 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79945.log.gz 2024-09-18T01:18:47.928 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79735.log.gz 2024-09-18T01:18:47.928 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114213.log.gz 2024-09-18T01:18:47.931 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32955.log.gz 2024-09-18T01:18:47.936 INFO:teuthology.orchestra.run.smithi049.stderr: 68.0% -- replaced with /var/log/ceph/cluster1-client.mirror.119564.log.gz 2024-09-18T01:18:47.937 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.88102.log.gz 2024-09-18T01:18:47.939 INFO:teuthology.orchestra.run.smithi049.stderr: 58.0% -- replaced with /var/log/ceph/cluster1-client.mirror.29686.log.gz 2024-09-18T01:18:47.942 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.127581.log.gz 2024-09-18T01:18:47.946 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65669.log.gz 2024-09-18T01:18:47.951 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68781.log.gz 2024-09-18T01:18:47.954 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66525.log.gz 2024-09-18T01:18:47.956 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67936.log.gz 2024-09-18T01:18:47.959 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.64936.log.gz 2024-09-18T01:18:47.962 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79049.log.gz 2024-09-18T01:18:47.965 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-client.admin.61763.log.gz 2024-09-18T01:18:47.966 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.139309.log.gz 2024-09-18T01:18:47.968 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.139653.log.gz 2024-09-18T01:18:47.968 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136612.log.gz 2024-09-18T01:18:47.968 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.134717.log.gz 2024-09-18T01:18:47.971 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.50740.log.gz 2024-09-18T01:18:47.971 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.61979.log.gz 2024-09-18T01:18:47.975 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster1-client.admin.97664.log.gz 2024-09-18T01:18:47.978 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.62784.log.gz 2024-09-18T01:18:47.982 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89025.log.gz 2024-09-18T01:18:47.985 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33312.log.gz 2024-09-18T01:18:47.990 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108074.log.gz 2024-09-18T01:18:47.995 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.120863.log.gz 2024-09-18T01:18:47.995 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79900.log.gz 2024-09-18T01:18:47.998 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.132178.log.gz 2024-09-18T01:18:48.000 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132558.log.gz 2024-09-18T01:18:48.004 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.129011.log.gz 2024-09-18T01:18:48.011 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.139005.log.gz 2024-09-18T01:18:48.020 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.139959.log.gz 2024-09-18T01:18:48.030 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.134418.log.gz 2024-09-18T01:18:48.033 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124556.log.gz 2024-09-18T01:18:48.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122015.log.gz 2024-09-18T01:18:48.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122142.log.gz 2024-09-18T01:18:48.046 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100974.log.gz 2024-09-18T01:18:48.049 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44682.log.gz 2024-09-18T01:18:48.056 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.73795.log.gz 2024-09-18T01:18:48.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103039.log.gz 2024-09-18T01:18:48.060 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110991.log.gz 2024-09-18T01:18:48.060 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.72513.log.gz 2024-09-18T01:18:48.064 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57664.log.gz 2024-09-18T01:18:48.064 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57690.log.gz 2024-09-18T01:18:48.068 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56678.log.gz 2024-09-18T01:18:48.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56553.log.gz 2024-09-18T01:18:48.074 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57877.log.gz 2024-09-18T01:18:48.076 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.108716.log.gz 2024-09-18T01:18:48.076 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54463.log.gz 2024-09-18T01:18:48.078 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.118786.log.gz 2024-09-18T01:18:48.082 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82612.log.gz 2024-09-18T01:18:48.083 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46936.log.gz 2024-09-18T01:18:48.085 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.60389.log.gz 2024-09-18T01:18:48.087 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.81161.log.gz 2024-09-18T01:18:48.091 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53981.log.gz 2024-09-18T01:18:48.093 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58196.log.gz 2024-09-18T01:18:48.096 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103925.log.gz 2024-09-18T01:18:48.100 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.62042.log.gz 2024-09-18T01:18:48.108 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34363.log.gz 2024-09-18T01:18:48.116 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.102150.log.gz 2024-09-18T01:18:48.118 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100723.log.gz 2024-09-18T01:18:48.119 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.129133.log.gz 2024-09-18T01:18:48.120 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115539.log.gz 2024-09-18T01:18:48.121 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101268.log.gz 2024-09-18T01:18:48.123 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103376.log.gz 2024-09-18T01:18:48.127 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster1-client.admin.120752.log.gz 2024-09-18T01:18:48.132 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122533.log.gz 2024-09-18T01:18:48.137 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113576.log.gz 2024-09-18T01:18:48.142 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137665.log.gz 2024-09-18T01:18:48.142 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40397.log.gz 2024-09-18T01:18:48.143 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34889.log.gz 2024-09-18T01:18:48.146 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.135987.log.gz 2024-09-18T01:18:48.151 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33585.log.gz 2024-09-18T01:18:48.154 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122144.log.gz 2024-09-18T01:18:48.157 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125811.log.gz 2024-09-18T01:18:48.165 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.132108.log.gz 2024-09-18T01:18:48.177 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.71119.log.gz 2024-09-18T01:18:48.181 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.70989.log.gz 2024-09-18T01:18:48.183 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70547.log.gz 2024-09-18T01:18:48.188 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91066.log.gz 2024-09-18T01:18:48.189 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36463.log.gz 2024-09-18T01:18:48.191 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.85230.log.gz 2024-09-18T01:18:48.193 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77874.log.gz 2024-09-18T01:18:48.195 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.139071.log.gz 2024-09-18T01:18:48.197 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster1-client.admin.41408.log.gz 2024-09-18T01:18:48.202 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44934.log.gz 2024-09-18T01:18:48.206 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77456.log.gz 2024-09-18T01:18:48.206 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57817.log.gz 2024-09-18T01:18:48.207 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32219.log.gz 2024-09-18T01:18:48.209 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.32220.log.gz 2024-09-18T01:18:48.211 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87632.log.gz 2024-09-18T01:18:48.212 INFO:teuthology.orchestra.run.smithi049.stderr: 80.6% -- replaced with /var/log/ceph/cluster1-client.mirror.30440.log.gz 2024-09-18T01:18:48.214 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.69202.log.gz 2024-09-18T01:18:48.217 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35835.log.gz 2024-09-18T01:18:48.219 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83514.log.gz 2024-09-18T01:18:48.220 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83468.log.gz 2024-09-18T01:18:48.227 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59665.log.gz 2024-09-18T01:18:48.231 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59291.log.gz 2024-09-18T01:18:48.233 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53581.log.gz 2024-09-18T01:18:48.237 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124554.log.gz 2024-09-18T01:18:48.239 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.119108.log.gz 2024-09-18T01:18:48.244 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99715.log.gz 2024-09-18T01:18:48.246 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.50517.log.gz 2024-09-18T01:18:48.251 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56299.log.gz 2024-09-18T01:18:48.253 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.52935.log.gz 2024-09-18T01:18:48.254 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34530.log.gz 2024-09-18T01:18:48.258 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56273.log.gz 2024-09-18T01:18:48.258 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56298.log.gz 2024-09-18T01:18:48.264 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46516.log.gz 2024-09-18T01:18:48.266 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster1-client.admin.76216.log.gz 2024-09-18T01:18:48.271 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.43573.log.gz 2024-09-18T01:18:48.276 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99411.log.gz 2024-09-18T01:18:48.278 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99413.log.gz 2024-09-18T01:18:48.278 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33270.log.gz 2024-09-18T01:18:48.288 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.30302.log.gz 2024-09-18T01:18:48.288 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48577.log.gz 2024-09-18T01:18:48.289 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48092.log.gz 2024-09-18T01:18:48.293 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.131623.log.gz 2024-09-18T01:18:48.299 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.132149.log.gz 2024-09-18T01:18:48.302 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.113433.log.gz 2024-09-18T01:18:48.307 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133617.log.gz 2024-09-18T01:18:48.310 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137456.log.gz 2024-09-18T01:18:48.313 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137661.log.gz 2024-09-18T01:18:48.318 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77413.log.gz 2024-09-18T01:18:48.342 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33731.log.gz 2024-09-18T01:18:48.345 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.109066.log.gz 2024-09-18T01:18:48.347 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.51388.log.gz 2024-09-18T01:18:48.348 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.93709.log.gz 2024-09-18T01:18:48.353 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.85800.log.gz 2024-09-18T01:18:48.356 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78797.log.gz 2024-09-18T01:18:48.358 INFO:teuthology.orchestra.run.smithi049.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53664.log.gz 2024-09-18T01:18:48.359 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90137.log.gz 2024-09-18T01:18:48.362 INFO:teuthology.orchestra.run.smithi049.stderr: 66.6% -- replaced with /var/log/ceph/cluster1-client.mirror.85002.log.gz 2024-09-18T01:18:48.365 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89002.log.gz 2024-09-18T01:18:48.367 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91275.log.gz 2024-09-18T01:18:48.369 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77915.log.gz 2024-09-18T01:18:48.379 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.114635.log.gz 2024-09-18T01:18:48.381 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.97354.log.gz 2024-09-18T01:18:48.383 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103713.log.gz 2024-09-18T01:18:48.386 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70085.log.gz 2024-09-18T01:18:48.389 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66013.log.gz 2024-09-18T01:18:48.392 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63065.log.gz 2024-09-18T01:18:48.396 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103923.log.gz 2024-09-18T01:18:48.398 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.73021.log.gz 2024-09-18T01:18:48.401 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99672.log.gz 2024-09-18T01:18:48.418 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47146.log.gz 2024-09-18T01:18:48.421 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112796.log.gz 2024-09-18T01:18:48.425 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.107679.log.gz 2024-09-18T01:18:48.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122077.log.gz 2024-09-18T01:18:48.430 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124132.log.gz 2024-09-18T01:18:48.433 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121397.log.gz 2024-09-18T01:18:48.444 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55354.log.gz 2024-09-18T01:18:48.449 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67979.log.gz 2024-09-18T01:18:48.453 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68949.log.gz 2024-09-18T01:18:48.457 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51642.log.gz 2024-09-18T01:18:48.457 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster1-client.admin.119896.log.gz 2024-09-18T01:18:48.461 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster2-client.admin.39692.log.gz 2024-09-18T01:18:48.462 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112316.log.gz 2024-09-18T01:18:48.465 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104807.log.gz 2024-09-18T01:18:48.506 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47529.log.gz 2024-09-18T01:18:48.511 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.60390.log.gz 2024-09-18T01:18:48.512 INFO:teuthology.orchestra.run.smithi049.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.71612.log.gz 2024-09-18T01:18:48.512 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.71963.log.gz 2024-09-18T01:18:48.517 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35370.log.gz 2024-09-18T01:18:48.517 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86619.log.gz 2024-09-18T01:18:48.517 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.33836.log.gz 2024-09-18T01:18:48.518 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.109088.log.gz 2024-09-18T01:18:48.522 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.117491.log.gz 2024-09-18T01:18:48.523 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34048.log.gz 2024-09-18T01:18:48.526 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37201.log.gz 2024-09-18T01:18:48.529 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79135.log.gz 2024-09-18T01:18:48.529 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.40672.log.gz 2024-09-18T01:18:48.532 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.64438.log.gz 2024-09-18T01:18:48.535 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80994.log.gz 2024-09-18T01:18:48.538 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91823.log.gz 2024-09-18T01:18:48.543 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75286.log.gz 2024-09-18T01:18:48.544 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37942.log.gz 2024-09-18T01:18:48.545 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99456.log.gz 2024-09-18T01:18:48.547 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48578.log.gz 2024-09-18T01:18:48.548 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.56928.log.gz 2024-09-18T01:18:48.557 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55060.log.gz 2024-09-18T01:18:48.559 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69495.log.gz 2024-09-18T01:18:48.562 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92327.log.gz 2024-09-18T01:18:48.566 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.95422.log.gz 2024-09-18T01:18:48.573 INFO:teuthology.orchestra.run.smithi049.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.admin.130037.log.gz 2024-09-18T01:18:48.578 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70043.log.gz 2024-09-18T01:18:48.580 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40946.log.gz 2024-09-18T01:18:48.580 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65927.log.gz 2024-09-18T01:18:48.585 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82684.log.gz 2024-09-18T01:18:48.589 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59289.log.gz 2024-09-18T01:18:48.604 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115264.log.gz 2024-09-18T01:18:48.606 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54095.log.gz 2024-09-18T01:18:48.609 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45377.log.gz 2024-09-18T01:18:48.610 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48516.log.gz 2024-09-18T01:18:48.614 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57558.log.gz 2024-09-18T01:18:48.615 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34887.log.gz 2024-09-18T01:18:48.615 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57731.log.gz 2024-09-18T01:18:48.619 INFO:teuthology.orchestra.run.smithi049.stderr: 66.6% -- replaced with /var/log/ceph/cluster1-client.mirror.119450.log.gz 2024-09-18T01:18:48.624 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122579.log.gz 2024-09-18T01:18:48.626 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.110891.log.gz 2024-09-18T01:18:48.638 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79603.log.gz 2024-09-18T01:18:48.641 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74096.log.gz 2024-09-18T01:18:48.645 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.51274.log.gz 2024-09-18T01:18:48.647 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.62355.log.gz 2024-09-18T01:18:48.649 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97163.log.gz 2024-09-18T01:18:48.651 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.38396.log.gz 2024-09-18T01:18:48.653 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92453.log.gz 2024-09-18T01:18:48.655 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.82085.log.gz 2024-09-18T01:18:48.656 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.38254.log.gz 2024-09-18T01:18:48.658 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.84098.log.gz 2024-09-18T01:18:48.659 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.84559.log.gz 2024-09-18T01:18:48.668 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92119.log.gz 2024-09-18T01:18:48.669 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92202.log.gz 2024-09-18T01:18:48.672 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.93399.log.gz 2024-09-18T01:18:48.678 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88665.log.gz 2024-09-18T01:18:48.681 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.86064.log.gz 2024-09-18T01:18:48.683 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79007.log.gz 2024-09-18T01:18:48.683 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68314.log.gz 2024-09-18T01:18:48.687 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78588.log.gz 2024-09-18T01:18:48.688 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42000.log.gz 2024-09-18T01:18:48.693 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58029.log.gz 2024-09-18T01:18:48.701 INFO:teuthology.orchestra.run.smithi049.stderr: 67.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51227.log.gz 2024-09-18T01:18:48.706 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58509.log.gz 2024-09-18T01:18:48.707 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66886.log.gz 2024-09-18T01:18:48.710 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.64802.log.gz 2024-09-18T01:18:48.710 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.54529.log.gz 2024-09-18T01:18:48.719 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115116.log.gz 2024-09-18T01:18:48.725 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67726.log.gz 2024-09-18T01:18:48.725 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122534.log.gz 2024-09-18T01:18:48.728 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66318.log.gz 2024-09-18T01:18:48.731 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59879.log.gz 2024-09-18T01:18:48.733 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134920.log.gz 2024-09-18T01:18:48.733 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.140614.log.gz 2024-09-18T01:18:48.737 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45946.log.gz 2024-09-18T01:18:48.742 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37600.log.gz 2024-09-18T01:18:48.745 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69285.log.gz 2024-09-18T01:18:48.750 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81707.log.gz 2024-09-18T01:18:48.753 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster1-client.admin.108670.log.gz 2024-09-18T01:18:48.754 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster1-client.admin.109698.log.gz 2024-09-18T01:18:48.761 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133827.log.gz 2024-09-18T01:18:48.783 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.131383.log.gz 2024-09-18T01:18:48.787 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125940.log.gz 2024-09-18T01:18:48.794 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.83723.log.gz 2024-09-18T01:18:48.798 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92116.log.gz 2024-09-18T01:18:48.832 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113745.log.gz 2024-09-18T01:18:48.835 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130446.log.gz 2024-09-18T01:18:48.842 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79303.log.gz 2024-09-18T01:18:48.845 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.116998.log.gz 2024-09-18T01:18:48.847 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115179.log.gz 2024-09-18T01:18:48.851 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103165.log.gz 2024-09-18T01:18:48.853 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103714.log.gz 2024-09-18T01:18:48.857 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103545.log.gz 2024-09-18T01:18:48.862 INFO:teuthology.orchestra.run.smithi049.stderr: 67.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74117.log.gz 2024-09-18T01:18:48.871 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.106280.log.gz 2024-09-18T01:18:48.873 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122099.log.gz 2024-09-18T01:18:48.876 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113072.log.gz 2024-09-18T01:18:48.878 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110225.log.gz 2024-09-18T01:18:48.882 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69245.log.gz 2024-09-18T01:18:48.886 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.91740.log.gz 2024-09-18T01:18:48.889 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.60247.log.gz 2024-09-18T01:18:48.893 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.52218.log.gz 2024-09-18T01:18:48.898 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37411.log.gz 2024-09-18T01:18:48.900 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.120114.log.gz 2024-09-18T01:18:48.902 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102533.log.gz 2024-09-18T01:18:48.903 INFO:teuthology.orchestra.run.smithi049.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51756.log.gz 2024-09-18T01:18:48.910 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36149.log.gz 2024-09-18T01:18:48.913 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.48864.log.gz 2024-09-18T01:18:48.920 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33354.log.gz 2024-09-18T01:18:48.923 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.36886.log.gz 2024-09-18T01:18:48.925 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77708.log.gz 2024-09-18T01:18:48.927 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52118.log.gz 2024-09-18T01:18:48.937 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.107431.log.gz 2024-09-18T01:18:48.940 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32003.log.gz 2024-09-18T01:18:48.941 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.95959.log.gz 2024-09-18T01:18:48.944 INFO:teuthology.orchestra.run.smithi049.stderr: 66.6% -- replaced with /var/log/ceph/cluster1-client.mirror.40739.log.gz 2024-09-18T01:18:48.944 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.87940.log.gz 2024-09-18T01:18:48.949 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.81372.log.gz 2024-09-18T01:18:48.954 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96613.log.gz 2024-09-18T01:18:48.956 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100052.log.gz 2024-09-18T01:18:48.957 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34257.log.gz 2024-09-18T01:18:48.959 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58031.log.gz 2024-09-18T01:18:48.964 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89865.log.gz 2024-09-18T01:18:48.965 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.109941.log.gz 2024-09-18T01:18:48.969 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.50645.log.gz 2024-09-18T01:18:48.973 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53242.log.gz 2024-09-18T01:18:48.973 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53279.log.gz 2024-09-18T01:18:48.978 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.59709.log.gz 2024-09-18T01:18:48.978 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.62442.log.gz 2024-09-18T01:18:48.979 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65760.log.gz 2024-09-18T01:18:48.984 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113111.log.gz 2024-09-18T01:18:48.989 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.116148.log.gz 2024-09-18T01:18:48.993 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.99369.log.gz 2024-09-18T01:18:48.996 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34149.log.gz 2024-09-18T01:18:49.001 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48368.log.gz 2024-09-18T01:18:49.003 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45587.log.gz 2024-09-18T01:18:49.015 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.111639.log.gz 2024-09-18T01:18:49.023 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115684.log.gz 2024-09-18T01:18:49.031 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52909.log.gz 2024-09-18T01:18:49.032 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103343.log.gz 2024-09-18T01:18:49.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54614.log.gz 2024-09-18T01:18:49.041 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.42512.log.gz 2024-09-18T01:18:49.042 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33689.log.gz 2024-09-18T01:18:49.045 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81919.log.gz 2024-09-18T01:18:49.047 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.31025.log.gz 2024-09-18T01:18:49.048 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.40330.log.gz 2024-09-18T01:18:49.053 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43530.log.gz 2024-09-18T01:18:49.053 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52195.log.gz 2024-09-18T01:18:49.054 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34258.log.gz 2024-09-18T01:18:49.062 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.60570.log.gz 2024-09-18T01:18:49.065 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.111893.log.gz 2024-09-18T01:18:49.076 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126715.log.gz 2024-09-18T01:18:49.084 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.83202.log.gz 2024-09-18T01:18:49.084 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133700.log.gz 2024-09-18T01:18:49.085 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133699.log.gz 2024-09-18T01:18:49.086 INFO:teuthology.orchestra.run.smithi049.stderr: 90.8% -- replaced with /var/log/ceph/cluster1-client.admin.85733.log.gz 2024-09-18T01:18:49.086 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.128697.log.gz 2024-09-18T01:18:49.087 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.109939.log.gz 2024-09-18T01:18:49.097 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.125728.log.gz 2024-09-18T01:18:49.114 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.57815.log.gz 2024-09-18T01:18:49.142 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104742.log.gz 2024-09-18T01:18:49.156 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.92747.log.gz 2024-09-18T01:18:49.158 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101058.log.gz 2024-09-18T01:18:49.158 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47207.log.gz 2024-09-18T01:18:49.159 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57619.log.gz 2024-09-18T01:18:49.161 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42575.log.gz 2024-09-18T01:18:49.163 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.49213.log.gz 2024-09-18T01:18:49.164 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89653.log.gz 2024-09-18T01:18:49.165 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.117019.log.gz 2024-09-18T01:18:49.168 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82638.log.gz 2024-09-18T01:18:49.172 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80194.log.gz 2024-09-18T01:18:49.175 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80365.log.gz 2024-09-18T01:18:49.177 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52324.log.gz 2024-09-18T01:18:49.180 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53106.log.gz 2024-09-18T01:18:49.183 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78923.log.gz 2024-09-18T01:18:49.192 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.102407.log.gz 2024-09-18T01:18:49.200 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65160.log.gz 2024-09-18T01:18:49.202 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71318.log.gz 2024-09-18T01:18:49.206 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66609.log.gz 2024-09-18T01:18:49.207 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34950.log.gz 2024-09-18T01:18:49.208 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67178.log.gz 2024-09-18T01:18:49.213 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.58745.log.gz 2024-09-18T01:18:49.218 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.87874.log.gz 2024-09-18T01:18:49.226 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.42358.log.gz 2024-09-18T01:18:49.226 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.42287.log.gz 2024-09-18T01:18:49.228 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.51683.log.gz 2024-09-18T01:18:49.230 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37303.log.gz 2024-09-18T01:18:49.232 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.72571.log.gz 2024-09-18T01:18:49.234 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78377.log.gz 2024-09-18T01:18:49.235 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80323.log.gz 2024-09-18T01:18:49.235 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81958.log.gz 2024-09-18T01:18:49.240 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82254.log.gz 2024-09-18T01:18:49.241 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.109041.log.gz 2024-09-18T01:18:49.243 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.94805.log.gz 2024-09-18T01:18:49.244 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.87358.log.gz 2024-09-18T01:18:49.248 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77999.log.gz 2024-09-18T01:18:49.254 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41081.log.gz 2024-09-18T01:18:49.259 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114486.log.gz 2024-09-18T01:18:49.261 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44473.log.gz 2024-09-18T01:18:49.262 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103966.log.gz 2024-09-18T01:18:49.263 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68568.log.gz 2024-09-18T01:18:49.273 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103040.log.gz 2024-09-18T01:18:49.283 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.101477.log.gz 2024-09-18T01:18:49.286 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100890.log.gz 2024-09-18T01:18:49.292 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54009.log.gz 2024-09-18T01:18:49.296 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.60116.log.gz 2024-09-18T01:18:49.297 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.53437.log.gz 2024-09-18T01:18:49.302 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99755.log.gz 2024-09-18T01:18:49.303 INFO:teuthology.orchestra.run.smithi049.stderr: 67.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129990.log.gz 2024-09-18T01:18:49.309 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54404.log.gz 2024-09-18T01:18:49.310 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55796.log.gz 2024-09-18T01:18:49.315 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68653.log.gz 2024-09-18T01:18:49.318 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79388.log.gz 2024-09-18T01:18:49.319 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.117043.log.gz 2024-09-18T01:18:49.325 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115956.log.gz 2024-09-18T01:18:49.331 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113960.log.gz 2024-09-18T01:18:49.343 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124636.log.gz 2024-09-18T01:18:49.345 INFO:teuthology.orchestra.run.smithi049.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.130721.log.gz 2024-09-18T01:18:49.346 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.123630.log.gz 2024-09-18T01:18:49.348 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134881.log.gz 2024-09-18T01:18:49.350 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123375.log.gz 2024-09-18T01:18:49.354 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.127178.log.gz 2024-09-18T01:18:49.357 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.127483.log.gz 2024-09-18T01:18:49.360 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136991.log.gz 2024-09-18T01:18:49.372 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137660.log.gz 2024-09-18T01:18:49.470 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.139871.log.gz 2024-09-18T01:18:49.478 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71379.log.gz 2024-09-18T01:18:49.481 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67975.log.gz 2024-09-18T01:18:49.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35898.log.gz 2024-09-18T01:18:49.489 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.88245.log.gz 2024-09-18T01:18:49.491 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89799.log.gz 2024-09-18T01:18:49.494 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81036.log.gz 2024-09-18T01:18:49.499 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80784.log.gz 2024-09-18T01:18:49.505 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37345.log.gz 2024-09-18T01:18:49.509 INFO:teuthology.orchestra.run.smithi049.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.38493.log.gz 2024-09-18T01:18:49.512 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55688.log.gz 2024-09-18T01:18:49.513 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56614.log.gz 2024-09-18T01:18:49.517 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.93966.log.gz 2024-09-18T01:18:49.520 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.127783.log.gz 2024-09-18T01:18:49.524 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70502.log.gz 2024-09-18T01:18:49.528 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103772.log.gz 2024-09-18T01:18:49.537 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45461.log.gz 2024-09-18T01:18:49.537 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45522.log.gz 2024-09-18T01:18:49.551 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.120663.log.gz 2024-09-18T01:18:49.558 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126569.log.gz 2024-09-18T01:18:49.560 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.129721.log.gz 2024-09-18T01:18:49.571 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122218.log.gz 2024-09-18T01:18:49.578 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.139208.log.gz 2024-09-18T01:18:49.595 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.95183.log.gz 2024-09-18T01:18:49.599 INFO:teuthology.orchestra.run.smithi049.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51072.log.gz 2024-09-18T01:18:49.628 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34467.log.gz 2024-09-18T01:18:49.634 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37158.log.gz 2024-09-18T01:18:49.638 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77455.log.gz 2024-09-18T01:18:49.642 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.87357.log.gz 2024-09-18T01:18:49.643 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84195.log.gz 2024-09-18T01:18:49.643 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.85934.log.gz 2024-09-18T01:18:49.645 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77977.log.gz 2024-09-18T01:18:49.646 INFO:teuthology.orchestra.run.smithi049.stderr: 67.8% -- replaced with /var/log/ceph/cluster1-client.mirror.96520.log.gz 2024-09-18T01:18:49.648 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.87095.log.gz 2024-09-18T01:18:49.648 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.87143.log.gz 2024-09-18T01:18:49.651 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.82381.log.gz 2024-09-18T01:18:49.654 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35900.log.gz 2024-09-18T01:18:49.655 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.63240.log.gz 2024-09-18T01:18:49.656 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40718.log.gz 2024-09-18T01:18:49.659 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.94011.log.gz 2024-09-18T01:18:49.662 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99967.log.gz 2024-09-18T01:18:49.664 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.91950.log.gz 2024-09-18T01:18:49.666 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.98575.log.gz 2024-09-18T01:18:49.668 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103586.log.gz 2024-09-18T01:18:49.676 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48996.log.gz 2024-09-18T01:18:49.678 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.107478.log.gz 2024-09-18T01:18:49.682 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91194.log.gz 2024-09-18T01:18:49.688 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35265.log.gz 2024-09-18T01:18:49.692 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67818.log.gz 2024-09-18T01:18:49.695 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77005.log.gz 2024-09-18T01:18:49.703 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.129022.log.gz 2024-09-18T01:18:49.703 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99797.log.gz 2024-09-18T01:18:49.705 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58322.log.gz 2024-09-18T01:18:49.709 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70545.log.gz 2024-09-18T01:18:49.712 INFO:teuthology.orchestra.run.smithi049.stderr: 67.3% -- replaced with /var/log/ceph/cluster1-client.mirror.96862.log.gz 2024-09-18T01:18:49.714 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137030.log.gz 2024-09-18T01:18:49.717 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48838.log.gz 2024-09-18T01:18:49.730 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58826.log.gz 2024-09-18T01:18:49.734 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster2-client.admin.62206.log.gz 2024-09-18T01:18:49.745 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.109353.log.gz 2024-09-18T01:18:49.754 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.106442.log.gz 2024-09-18T01:18:49.757 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108188.log.gz 2024-09-18T01:18:49.762 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.130973.log.gz 2024-09-18T01:18:49.763 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133366.log.gz 2024-09-18T01:18:49.765 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.132940.log.gz 2024-09-18T01:18:49.767 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.135730.log.gz 2024-09-18T01:18:49.786 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126862.log.gz 2024-09-18T01:18:49.789 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124972.log.gz 2024-09-18T01:18:49.794 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.116000.log.gz 2024-09-18T01:18:49.796 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.117971.log.gz 2024-09-18T01:18:49.802 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.51160.log.gz 2024-09-18T01:18:49.809 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90136.log.gz 2024-09-18T01:18:49.812 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103251.log.gz 2024-09-18T01:18:49.817 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34402.log.gz 2024-09-18T01:18:49.819 INFO:teuthology.orchestra.run.smithi049.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31696.log.gz 2024-09-18T01:18:49.821 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31687.log.gz 2024-09-18T01:18:49.823 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88916.log.gz 2024-09-18T01:18:49.827 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80950.log.gz 2024-09-18T01:18:49.832 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77623.log.gz 2024-09-18T01:18:49.833 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111683.log.gz 2024-09-18T01:18:49.835 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32788.log.gz 2024-09-18T01:18:49.836 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster2-client.admin.39401.log.gz 2024-09-18T01:18:49.839 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.40074.log.gz 2024-09-18T01:18:49.839 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86549.log.gz 2024-09-18T01:18:49.844 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65475.log.gz 2024-09-18T01:18:49.848 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35622.log.gz 2024-09-18T01:18:49.851 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.72118.log.gz 2024-09-18T01:18:49.856 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69136.log.gz 2024-09-18T01:18:49.860 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69580.log.gz 2024-09-18T01:18:49.863 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.56887.log.gz 2024-09-18T01:18:49.871 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57138.log.gz 2024-09-18T01:18:49.887 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.115262.log.gz 2024-09-18T01:18:49.889 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111537.log.gz 2024-09-18T01:18:49.889 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137662.log.gz 2024-09-18T01:18:49.896 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48409.log.gz 2024-09-18T01:18:49.900 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47104.log.gz 2024-09-18T01:18:49.906 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36781.log.gz 2024-09-18T01:18:49.912 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68992.log.gz 2024-09-18T01:18:49.915 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66055.log.gz 2024-09-18T01:18:49.920 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31604.log.gz 2024-09-18T01:18:49.922 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37092.log.gz 2024-09-18T01:18:49.926 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.38797.log.gz 2024-09-18T01:18:49.929 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39369.log.gz 2024-09-18T01:18:49.931 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33519.log.gz 2024-09-18T01:18:49.939 INFO:teuthology.orchestra.run.smithi049.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-client.admin.51730.log.gz 2024-09-18T01:18:49.939 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.40584.log.gz 2024-09-18T01:18:49.944 INFO:teuthology.orchestra.run.smithi049.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.29977.log.gz 2024-09-18T01:18:49.949 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77557.log.gz 2024-09-18T01:18:49.956 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.84225.log.gz 2024-09-18T01:18:49.959 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.93917.log.gz 2024-09-18T01:18:49.961 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.95243.log.gz 2024-09-18T01:18:49.965 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100176.log.gz 2024-09-18T01:18:49.966 INFO:teuthology.orchestra.run.smithi049.stderr: 80.5% -- replaced with /var/log/ceph/cluster1-client.mirror.64021.log.gz 2024-09-18T01:18:49.971 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115896.log.gz 2024-09-18T01:18:49.976 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100637.log.gz 2024-09-18T01:18:49.981 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104721.log.gz 2024-09-18T01:18:49.995 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126990.log.gz 2024-09-18T01:18:49.997 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134288.log.gz 2024-09-18T01:18:50.000 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138317.log.gz 2024-09-18T01:18:50.020 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126782.log.gz 2024-09-18T01:18:50.027 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.125476.log.gz 2024-09-18T01:18:50.047 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111475.log.gz 2024-09-18T01:18:50.048 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.128177.log.gz 2024-09-18T01:18:50.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46641.log.gz 2024-09-18T01:18:50.057 INFO:teuthology.orchestra.run.smithi049.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.mirror.131912.log.gz 2024-09-18T01:18:50.057 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.131817.log.gz 2024-09-18T01:18:50.061 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115537.log.gz 2024-09-18T01:18:50.062 INFO:teuthology.orchestra.run.smithi049.stderr: 66.6% -- replaced with /var/log/ceph/cluster1-client.mirror.118766.log.gz 2024-09-18T01:18:50.066 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44622.log.gz 2024-09-18T01:18:50.069 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112650.log.gz 2024-09-18T01:18:50.075 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.105095.log.gz 2024-09-18T01:18:50.075 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.134922.log.gz 2024-09-18T01:18:50.078 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79561.log.gz 2024-09-18T01:18:50.082 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114001.log.gz 2024-09-18T01:18:50.087 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.132407.log.gz 2024-09-18T01:18:50.089 INFO:teuthology.orchestra.run.smithi049.stderr: 56.3% -- replaced with /var/log/ceph/cluster1-client.mirror.29720.log.gz 2024-09-18T01:18:50.090 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135472.log.gz 2024-09-18T01:18:50.093 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.135769.log.gz 2024-09-18T01:18:50.097 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137240.log.gz 2024-09-18T01:18:50.102 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124739.log.gz 2024-09-18T01:18:50.111 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126447.log.gz 2024-09-18T01:18:50.113 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.129668.log.gz 2024-09-18T01:18:50.117 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.129269.log.gz 2024-09-18T01:18:50.131 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84426.log.gz 2024-09-18T01:18:50.148 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52738.log.gz 2024-09-18T01:18:50.172 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.120250.log.gz 2024-09-18T01:18:50.189 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67891.log.gz 2024-09-18T01:18:50.197 INFO:teuthology.orchestra.run.smithi049.stderr: 18.2% -- replaced with /var/log/ceph/cluster1-client.admin.97526.log.gz 2024-09-18T01:18:50.200 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46683.log.gz 2024-09-18T01:18:50.201 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89235.log.gz 2024-09-18T01:18:50.205 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.30340.log.gz 2024-09-18T01:18:50.208 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90726.log.gz 2024-09-18T01:18:50.211 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.94932.log.gz 2024-09-18T01:18:50.213 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40261.log.gz 2024-09-18T01:18:50.218 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33215.log.gz 2024-09-18T01:18:50.221 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35941.log.gz 2024-09-18T01:18:50.224 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.93684.log.gz 2024-09-18T01:18:50.224 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.80449.log.gz 2024-09-18T01:18:50.224 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.49165.log.gz 2024-09-18T01:18:50.228 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.50005.log.gz 2024-09-18T01:18:50.233 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88454.log.gz 2024-09-18T01:18:50.233 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59769.log.gz 2024-09-18T01:18:50.248 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101478.log.gz 2024-09-18T01:18:50.252 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46305.log.gz 2024-09-18T01:18:50.256 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101310.log.gz 2024-09-18T01:18:50.257 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107846.log.gz 2024-09-18T01:18:50.261 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48369.log.gz 2024-09-18T01:18:50.265 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.99201.log.gz 2024-09-18T01:18:50.275 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76170.log.gz 2024-09-18T01:18:50.280 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115600.log.gz 2024-09-18T01:18:50.283 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114381.log.gz 2024-09-18T01:18:50.284 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115119.log.gz 2024-09-18T01:18:50.293 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114171.log.gz 2024-09-18T01:18:50.299 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115809.log.gz 2024-09-18T01:18:50.302 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.114759.log.gz 2024-09-18T01:18:50.308 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.135689.log.gz 2024-09-18T01:18:50.317 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110504.log.gz 2024-09-18T01:18:50.318 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.135555.log.gz 2024-09-18T01:18:50.319 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137242.log.gz 2024-09-18T01:18:50.321 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.118813.log.gz 2024-09-18T01:18:50.327 INFO:teuthology.orchestra.run.smithi049.stderr: 90.8% -- replaced with /var/log/ceph/cluster1-client.admin.108162.log.gz 2024-09-18T01:18:50.340 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.76811.log.gz 2024-09-18T01:18:50.351 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.126653.log.gz 2024-09-18T01:18:50.354 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.123059.log.gz 2024-09-18T01:18:50.356 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.121582.log.gz 2024-09-18T01:18:50.369 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77203.log.gz 2024-09-18T01:18:50.407 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136195.log.gz 2024-09-18T01:18:50.413 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58786.log.gz 2024-09-18T01:18:50.414 INFO:teuthology.orchestra.run.smithi049.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.63705.log.gz 2024-09-18T01:18:50.418 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50842.log.gz 2024-09-18T01:18:50.421 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56822.log.gz 2024-09-18T01:18:50.423 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56972.log.gz 2024-09-18T01:18:50.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77790.log.gz 2024-09-18T01:18:50.428 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92246.log.gz 2024-09-18T01:18:50.430 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.97503.log.gz 2024-09-18T01:18:50.445 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.83249.log.gz 2024-09-18T01:18:50.454 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65585.log.gz 2024-09-18T01:18:50.457 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111099.log.gz 2024-09-18T01:18:50.463 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46011.log.gz 2024-09-18T01:18:50.471 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100049.log.gz 2024-09-18T01:18:50.484 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133321.log.gz 2024-09-18T01:18:50.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101814.log.gz 2024-09-18T01:18:50.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101815.log.gz 2024-09-18T01:18:50.500 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster1-client.admin.131669.log.gz 2024-09-18T01:18:50.510 INFO:teuthology.orchestra.run.smithi028.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2024-09-18T01:18:50.514 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133532.log.gz 2024-09-18T01:18:50.529 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.139359.log.gz 2024-09-18T01:18:50.543 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.39270.log.gz 2024-09-18T01:18:50.562 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.60678.log.gz 2024-09-18T01:18:50.568 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.83301.log.gz 2024-09-18T01:18:50.570 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.85022.log.gz 2024-09-18T01:18:50.571 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88455.log.gz 2024-09-18T01:18:50.572 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88959.log.gz 2024-09-18T01:18:50.573 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.52347.log.gz 2024-09-18T01:18:50.574 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.139239.log.gz 2024-09-18T01:18:50.581 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.106529.log.gz 2024-09-18T01:18:50.584 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.72032.log.gz 2024-09-18T01:18:50.589 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81286.log.gz 2024-09-18T01:18:50.592 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81457.log.gz 2024-09-18T01:18:50.594 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47882.log.gz 2024-09-18T01:18:50.599 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34685.log.gz 2024-09-18T01:18:50.600 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34779.log.gz 2024-09-18T01:18:50.604 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103882.log.gz 2024-09-18T01:18:50.605 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.107176.log.gz 2024-09-18T01:18:50.613 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101226.log.gz 2024-09-18T01:18:50.617 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104871.log.gz 2024-09-18T01:18:50.618 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112522.log.gz 2024-09-18T01:18:50.629 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster1-client.admin.120214.log.gz 2024-09-18T01:18:50.633 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.110060.log.gz 2024-09-18T01:18:50.637 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110314.log.gz 2024-09-18T01:18:50.647 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68611.log.gz 2024-09-18T01:18:50.648 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68713.log.gz 2024-09-18T01:18:50.649 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster2-client.admin.117700.log.gz 2024-09-18T01:18:50.650 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster1-client.admin.131844.log.gz 2024-09-18T01:18:50.673 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.50104.log.gz 2024-09-18T01:18:50.688 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.140214.log.gz 2024-09-18T01:18:50.701 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.39262.log.gz 2024-09-18T01:18:50.731 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57474.log.gz 2024-09-18T01:18:50.733 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35369.log.gz 2024-09-18T01:18:50.736 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78505.log.gz 2024-09-18T01:18:50.736 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.30581.log.gz 2024-09-18T01:18:50.742 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.95681.log.gz 2024-09-18T01:18:50.746 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.98392.log.gz 2024-09-18T01:18:50.746 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.98417.log.gz 2024-09-18T01:18:50.748 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.40188.log.gz 2024-09-18T01:18:50.748 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.72119.log.gz 2024-09-18T01:18:50.754 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55689.log.gz 2024-09-18T01:18:50.758 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65129.log.gz 2024-09-18T01:18:50.759 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.110146.log.gz 2024-09-18T01:18:50.759 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110289.log.gz 2024-09-18T01:18:50.762 INFO:teuthology.orchestra.run.smithi049.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.87804.log.gz 2024-09-18T01:18:50.766 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79177.log.gz 2024-09-18T01:18:50.766 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78924.log.gz 2024-09-18T01:18:50.769 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.40853.log.gz 2024-09-18T01:18:50.770 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.108249.log.gz 2024-09-18T01:18:50.774 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.31266.log.gz 2024-09-18T01:18:50.785 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68863.log.gz 2024-09-18T01:18:50.788 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.75606.log.gz 2024-09-18T01:18:50.790 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70337.log.gz 2024-09-18T01:18:50.803 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.107565.log.gz 2024-09-18T01:18:50.808 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster1-client.admin.132430.log.gz 2024-09-18T01:18:50.817 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47248.log.gz 2024-09-18T01:18:50.823 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59375.log.gz 2024-09-18T01:18:50.831 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122578.log.gz 2024-09-18T01:18:50.835 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.130466.log.gz 2024-09-18T01:18:50.842 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.31217.log.gz 2024-09-18T01:18:50.850 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41456.log.gz 2024-09-18T01:18:50.853 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77285.log.gz 2024-09-18T01:18:50.876 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136571.log.gz 2024-09-18T01:18:50.877 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107752.log.gz 2024-09-18T01:18:50.882 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113961.log.gz 2024-09-18T01:18:50.886 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135051.log.gz 2024-09-18T01:18:50.886 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59792.log.gz 2024-09-18T01:18:50.894 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.131754.log.gz 2024-09-18T01:18:50.905 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.32536.log.gz 2024-09-18T01:18:50.908 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36990.log.gz 2024-09-18T01:18:50.909 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.95686.log.gz 2024-09-18T01:18:50.913 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80698.log.gz 2024-09-18T01:18:50.913 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104764.log.gz 2024-09-18T01:18:50.913 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78715.log.gz 2024-09-18T01:18:50.917 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99673.log.gz 2024-09-18T01:18:50.923 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.80278.log.gz 2024-09-18T01:18:50.928 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56468.log.gz 2024-09-18T01:18:50.930 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56637.log.gz 2024-09-18T01:18:50.934 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59582.log.gz 2024-09-18T01:18:50.940 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88328.log.gz 2024-09-18T01:18:50.953 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45672.log.gz 2024-09-18T01:18:50.962 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66483.log.gz 2024-09-18T01:18:50.964 INFO:teuthology.orchestra.run.smithi049.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73848.log.gz 2024-09-18T01:18:50.965 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73962.log.gz 2024-09-18T01:18:50.965 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74573.log.gz 2024-09-18T01:18:50.966 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.71056.log.gz 2024-09-18T01:18:50.966 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74532.log.gz 2024-09-18T01:18:50.975 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113859.log.gz 2024-09-18T01:18:50.975 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134247.log.gz 2024-09-18T01:18:50.978 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.119946.log.gz 2024-09-18T01:18:50.979 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.119983.log.gz 2024-09-18T01:18:50.984 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112103.log.gz 2024-09-18T01:18:50.998 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67095.log.gz 2024-09-18T01:18:50.999 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75128.log.gz 2024-09-18T01:18:51.002 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.30438.log.gz 2024-09-18T01:18:51.006 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.61680.log.gz 2024-09-18T01:18:51.010 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36633.log.gz 2024-09-18T01:18:51.013 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36737.log.gz 2024-09-18T01:18:51.016 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.106858.log.gz 2024-09-18T01:18:51.028 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125603.log.gz 2024-09-18T01:18:51.049 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81329.log.gz 2024-09-18T01:18:51.062 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132255.log.gz 2024-09-18T01:18:51.065 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124529.log.gz 2024-09-18T01:18:51.073 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.86812.log.gz 2024-09-18T01:18:51.074 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90685.log.gz 2024-09-18T01:18:51.076 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.83300.log.gz 2024-09-18T01:18:51.078 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103252.log.gz 2024-09-18T01:18:51.082 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93022.log.gz 2024-09-18T01:18:51.086 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35516.log.gz 2024-09-18T01:18:51.089 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88392.log.gz 2024-09-18T01:18:51.092 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89126.log.gz 2024-09-18T01:18:51.093 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.94957.log.gz 2024-09-18T01:18:51.094 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.31656.log.gz 2024-09-18T01:18:51.099 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77663.log.gz 2024-09-18T01:18:51.103 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.98133.log.gz 2024-09-18T01:18:51.108 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.116181.log.gz 2024-09-18T01:18:51.112 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.98753.log.gz 2024-09-18T01:18:51.113 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112060.log.gz 2024-09-18T01:18:51.124 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101354.log.gz 2024-09-18T01:18:51.124 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101395.log.gz 2024-09-18T01:18:51.132 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103082.log.gz 2024-09-18T01:18:51.135 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45479.log.gz 2024-09-18T01:18:51.150 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137877.log.gz 2024-09-18T01:18:51.153 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138128.log.gz 2024-09-18T01:18:51.155 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.106629.log.gz 2024-09-18T01:18:51.168 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33216.log.gz 2024-09-18T01:18:51.182 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.116779.log.gz 2024-09-18T01:18:51.183 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.118171.log.gz 2024-09-18T01:18:51.467 INFO:teuthology.orchestra.run.smithi028.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2024-09-18T01:18:58.950 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.49798.log.gz 2024-09-18T01:18:58.950 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.86047.log.gz 2024-09-18T01:18:58.950 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.92746.log.gz 2024-09-18T01:18:58.950 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.102955.log.gz 2024-09-18T01:18:58.950 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.41672.log.gz 2024-09-18T01:18:58.950 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.93085.log.gz 2024-09-18T01:18:58.950 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136610.log.gz 2024-09-18T01:18:58.950 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.33690.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.108021.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.57139.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.63448.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77048.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: 57.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29908.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138005.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.118839.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.97932.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43907.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103964.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44411.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.132469.log.gz 2024-09-18T01:18:58.951 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125579.log.gz 2024-09-18T01:18:58.952 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66757.log.gz 2024-09-18T01:18:58.952 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104300.log.gz 2024-09-18T01:18:58.952 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57818.log.gz 2024-09-18T01:18:58.952 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55183.log.gz 2024-09-18T01:18:58.952 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.140531.log.gz 2024-09-18T01:18:58.952 INFO:teuthology.orchestra.run.smithi049.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.72407.log.gz 2024-09-18T01:18:58.952 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45042.log.gz 2024-09-18T01:18:58.952 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster1-client.admin.41695.log.gz 2024-09-18T01:18:58.952 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.52828.log.gz 2024-09-18T01:18:58.952 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100175.log.gz 2024-09-18T01:18:58.952 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80364.log.gz 2024-09-18T01:18:58.952 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.126739.log.gz 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87670.log.gz 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 68.0% -- replaced with /var/log/ceph/cluster1-client.mirror.136990.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.85914.log.gz 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92831.log.gz 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 18.1% -- replaced with /var/log/ceph/cluster2-client.admin.139758.log.gz 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.121220.log.gz 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112378.log.gz 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.128101.log.gz 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68523.log.gz 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113577.log.gz 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.71561.log.gz 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74646.log.gz 2024-09-18T01:18:58.953 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32070.log.gz 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.106242.log.gz 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57055.log.gz 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70124.log.gz 65.6% 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.40511.log.gz 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76948.log.gz 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111809.log.gz 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130539.log.gz 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113429.log.gz 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91553.log.gz 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46092.log.gz 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54487.log.gz 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110503.log.gz 2024-09-18T01:18:58.954 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.109653.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.119811.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69452.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.71804.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111036.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42050.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78254.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.95238.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104594.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.62609.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.127503.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.56929.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37410.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80909.log.gz 2024-09-18T01:18:58.955 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.52025.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.134497.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80028.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.108849.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54850.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.93230.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.76126.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.72257.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.115011.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133287.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121866.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57307.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35032.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78966.log.gz 2024-09-18T01:18:58.956 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.30871.log.gz 2024-09-18T01:18:58.957 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80111.log.gz 2024-09-18T01:18:58.957 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.108851.log.gz 2024-09-18T01:18:58.957 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.127787.log.gz 2024-09-18T01:18:58.957 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130914.log.gz 2024-09-18T01:18:58.957 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68229.log.gz 2024-09-18T01:18:58.957 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.73424.log.gz 2024-09-18T01:18:58.957 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.76527.log.gz 2024-09-18T01:18:58.957 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126840.log.gz 2024-09-18T01:18:58.957 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.58321.log.gz 2024-09-18T01:18:58.957 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79901.log.gz 2024-09-18T01:18:58.957 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.62925.log.gz 2024-09-18T01:18:58.957 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122243.log.gz 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42742.log.gz 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.41828.log.gz 19.9% 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.129280.log.gz 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.114698.log.gz 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68654.log.gz 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92537.log.gz 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.64709.log.gz 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.62208.log.gz 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43487.log.gz 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112587.log.gz 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66527.log.gz 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.113431.log.gz 2024-09-18T01:18:58.958 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster2-client.admin.73018.log.gz 2024-09-18T01:18:58.959 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.85364.log.gz 2024-09-18T01:18:58.959 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137201.log.gz 2024-09-18T01:18:58.959 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99118.log.gz 2024-09-18T01:18:58.959 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.33846.log.gz 2024-09-18T01:18:58.959 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131974.log.gz 2024-09-18T01:18:58.959 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90473.log.gz 2024-09-18T01:18:58.959 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35264.log.gz 2024-09-18T01:18:58.959 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66885.log.gz 2024-09-18T01:18:58.959 INFO:teuthology.orchestra.run.smithi049.stderr: 66.1% -- replaced with /var/log/ceph/cluster1-client.mirror.85686.log.gz 2024-09-18T01:18:58.959 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114549.log.gz 2024-09-18T01:18:58.959 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.119734.log.gz 2024-09-18T01:18:58.959 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104048.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.62898.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.125058.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.51958.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122449.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.109177.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.105903.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107960.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68079.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41805.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47778.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111433.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53852.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56973.log.gz 2024-09-18T01:18:58.960 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.86573.log.gz 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72817.log.gz 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136570.log.gz 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.58662.log.gz 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.86837.log.gz 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134077.log.gz 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65266.log.gz 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123081.log.gz 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54317.log.gz 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.86877.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.58365.log.gz 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.63927.log.gz 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126233.log.gz 2024-09-18T01:18:58.961 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51870.log.gz 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70208.log.gz 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45167.log.gz 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43089.log.gz 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster1-client.admin.87270.log.gz 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.130352.log.gz 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68567.log.gz 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57899.log.gz 89.8% 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.77369.log.gz 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: 18.2% -- replaced with /var/log/ceph/cluster1-client.admin.87465.log.gz 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.119697.log.gz 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.97742.log.gz 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59035.log.gz 2024-09-18T01:18:58.962 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.116591.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.63381.log.gz 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58240.log.gz 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82560.log.gz 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130063.log.gz 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37515.log.gz 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.76515.log.gz 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101267.log.gz 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.102998.log.gz 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70755.log.gz 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.93293.log.gz 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.61375.log.gz 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.135343.log.gz 2024-09-18T01:18:58.963 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.49136.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137620.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.60878.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137791.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104259.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99219.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.128715.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.93805.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123711.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.60481.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.119067.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35307.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.51502.log.gz 2024-09-18T01:18:58.964 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.110335.log.gz 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.108585.log.gz 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53485.log.gz 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80656.log.gz 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.97110.log.gz 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster2-client.admin.106761.log.gz 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.61038.log.gz 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82910.log.gz 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52400.log.gz 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43531.log.gz 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.51475.log.gz 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124555.log.gz 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101687.log.gz 89.0% 2024-09-18T01:18:58.965 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.31797.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.112104.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.94559.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46640.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.130844.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135558.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.79986.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68228.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103839.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.73023.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.98246.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.admin.29601.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56718.log.gz 2024-09-18T01:18:58.966 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104135.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48159.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47420.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34475.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.64523.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85437.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71323.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47145.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.140067.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.116910.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.74880.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36572.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135302.log.gz 2024-09-18T01:18:58.967 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.105978.log.gz 2024-09-18T01:18:58.968 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.127651.log.gz 2024-09-18T01:18:58.968 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55878.log.gz 2024-09-18T01:18:58.968 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.30821.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.45252.log.gz 2024-09-18T01:18:58.968 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:58.968 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.39070.log.gz 2024-09-18T01:18:58.968 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.50795.log.gz 2024-09-18T01:18:58.968 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.50544.log.gz 2024-09-18T01:18:58.968 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.102916.log.gz 2024-09-18T01:18:58.968 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 96.3% -- replaced with /var/log/ceph/cluster1-client.mirror.58406.log.gz 2024-09-18T01:18:58.968 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.119429.log.gz 2024-09-18T01:18:58.968 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68442.log.gz 2024-09-18T01:18:58.968 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137457.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57771.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.97277.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36843.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.96426.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101897.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44620.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.84908.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.106443.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58111.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.124404.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97049.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.131191.log.gz 2024-09-18T01:18:58.969 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44094.log.gz 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88539.log.gz 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86327.log.gz 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 19.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43572.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.132214.log.gz 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.61820.log.gz 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82680.log.gz 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66189.log.gz 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66157.log.gz 89.9% 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.102702.log.gz 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78798.log.gz 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.108493.log.gz 89.9% 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.102532.log.gz 2024-09-18T01:18:58.970 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57900.log.gz 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.121656.log.gz 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.32175.log.gz 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37705.log.gz 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.121911.log.gz 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 18.2% -- replaced with /var/log/ceph/cluster1-client.admin.42464.log.gz 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45884.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.79479.log.gz 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.130104.log.gz 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.134715.log.gz 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122143.log.gz 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster1-client.admin.108536.log.gz 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.30304.log.gz 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.73404.log.gz 2024-09-18T01:18:58.971 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110546.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.76415.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.82738.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.121337.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103501.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.127580.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43316.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.139667.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.52267.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65804.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56402.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47061.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.78419.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.109216.log.gz 2024-09-18T01:18:58.972 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115180.log.gz 2024-09-18T01:18:58.973 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31962.log.gz 2024-09-18T01:18:58.973 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69917.log.gz 2024-09-18T01:18:58.973 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86393.log.gz 2024-09-18T01:18:58.973 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36569.log.gz 2024-09-18T01:18:58.973 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.56763.log.gz 2024-09-18T01:18:58.973 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102194.log.gz 2024-09-18T01:18:58.973 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104827.log.gz 2024-09-18T01:18:58.973 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54724.log.gz 2024-09-18T01:18:58.973 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.113222.log.gz 2024-09-18T01:18:58.973 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.53895.log.gz 2024-09-18T01:18:58.973 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.105785.log.gz 2024-09-18T01:18:58.973 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.129267.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82191.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.49668.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111474.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.52782.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110780.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80406.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122512.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81328.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65541.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.60501.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.85706.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54031.log.gz 2024-09-18T01:18:58.974 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66233.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56844.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100513.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.46473.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.139889.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 66.6% -- replaced with /var/log/ceph/cluster1-client.mirror.63220.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138193.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 68.0% -- replaced with /var/log/ceph/cluster1-client.mirror.96976.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.131048.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69704.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68227.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34634.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92790.log.gz 2024-09-18T01:18:58.975 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102831.log.gz 2024-09-18T01:18:58.976 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81624.log.gz 2024-09-18T01:18:58.976 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.97641.log.gz 2024-09-18T01:18:58.976 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112525.log.gz 2024-09-18T01:18:58.976 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37302.log.gz 2024-09-18T01:18:58.976 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135980.log.gz 2024-09-18T01:18:58.976 INFO:teuthology.orchestra.run.smithi049.stderr: 66.4% -- replaced with /var/log/ceph/cluster2-client.admin.29840.log.gz 2024-09-18T01:18:58.976 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.134501.log.gz 2024-09-18T01:18:58.976 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53069.log.gz 2024-09-18T01:18:58.976 INFO:teuthology.orchestra.run.smithi049.stderr: 20.6% -- replaced with /var/log/ceph/cluster2-client.admin.39697.log.gz 2024-09-18T01:18:58.976 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54360.log.gz 2024-09-18T01:18:58.976 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79817.log.gz 2024-09-18T01:18:58.976 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69346.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41588.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.118436.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.41651.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89505.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101228.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41779.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93166.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79984.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55563.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.63126.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.39909.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.126026.log.gz 2024-09-18T01:18:58.977 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90979.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.43275.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44118.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.121760.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 2.3% -- replaced with /var/log/ceph/cluster1-client.mirror.29669.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66526.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.75017.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.74620.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55393.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.119242.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56508.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90139.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42771.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43023.log.gz 2024-09-18T01:18:58.978 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88666.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32360.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.82296.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.126946.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56342.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.43315.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44412.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43109.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.129809.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.61827.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.75102.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45250.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133741.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122871.log.gz 2024-09-18T01:18:58.979 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.139360.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77709.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84400.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45102.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.39850.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.33626.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46262.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.120423.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 68.0% -- replaced with /var/log/ceph/cluster1-client.mirror.97204.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.51616.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.96251.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102618.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59414.log.gz 2024-09-18T01:18:58.980 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135260.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89295.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.105226.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.116934.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80531.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35306.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.41454.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51890.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.40038.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.76895.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster1-client.admin.30533.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.116043.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.40028.log.gz 2024-09-18T01:18:58.981 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.83949.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46364.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112167.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121784.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35099.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.114424.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.31264.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80069.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78587.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43418.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122367.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.36151.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.93756.log.gz 2024-09-18T01:18:58.982 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107390.log.gz 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.139666.log.gz 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47271.log.gz 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.40531.log.gz 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68821.log.gz 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.98657.log.gz 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51662.log.gz 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80112.log.gz 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92600.log.gz 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126571.log.gz 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.38395.log.gz 90.0% 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.103673.log.gz 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.118611.log.gz 2024-09-18T01:18:58.983 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112946.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.102915.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37661.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster1-client.admin.131337.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.119470.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103796.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.64075.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133868.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100428.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32744.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101057.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.104342.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43131.log.gz 2024-09-18T01:18:58.984 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59245.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.87184.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43049.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.73681.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.135982.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.117357.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89863.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99903.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65887.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68146.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103125.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36003.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.mirror.131385.log.gz 2024-09-18T01:18:58.985 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.120998.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster1-client.admin.130867.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37093.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35940.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.110889.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.52671.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72844.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74968.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118745.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% 89.9% -- replaced with /var/log/ceph/cluster2-client.admin.140278.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.104258.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93510.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53708.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121787.log.gz 2024-09-18T01:18:58.986 INFO:teuthology.orchestra.run.smithi049.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.86215.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.51113.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43951.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124319.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57054.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49070.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.84637.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42917.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67239.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81288.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.102107.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57610.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.100218.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.126361.log.gz 2024-09-18T01:18:58.987 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101605.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.98888.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.98611.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122788.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82044.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.139003.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45947.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104385.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91133.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92034.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35942.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69413.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.100847.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% 18.7% -- replaced with /var/log/ceph/cluster1-client.mirror.69916.log.gz -- replaced with /var/log/ceph/cluster2-client.admin.107201.log.gz 2024-09-18T01:18:58.988 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.120729.log.gz 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126630.log.gz 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131760.log.gz 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112102.log.gz 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.90920.log.gz 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122662.log 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91615.log.gz 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52866.log.gz 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.97579.log.gz 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.104093.log.gz 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79521.log.gz 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.76873.log.gz 2024-09-18T01:18:58.989 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.114969.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43048.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster2-client.admin.140689.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.70463.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112483.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46998.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.38801.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89507.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127030.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115540.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61155.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.41541.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113917.log.gz 2024-09-18T01:18:58.990 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.84355.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.39116.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45588.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.126991.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42927.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89125.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.94699.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.76614.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100259.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101144.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.47568.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.72628.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.116607.log.gz 2024-09-18T01:18:58.991 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51092.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35160.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100303.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.97318.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46406.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81834.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.95954.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124469.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.106444.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 67.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74687.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132106.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.109360.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80572.log.gz 2024-09-18T01:18:58.992 INFO:teuthology.orchestra.run.smithi049.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.mirror.109005.log.gz 2024-09-18T01:18:58.993 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67389.log.gz 2024-09-18T01:18:58.993 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47695.log.gz 2024-09-18T01:18:58.993 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138358.log.gz 2024-09-18T01:18:58.993 INFO:teuthology.orchestra.run.smithi049.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.120025.log.gz 2024-09-18T01:18:58.993 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35793.log.gz 2024-09-18T01:18:58.993 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93487.log.gz 2024-09-18T01:18:58.993 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.72117.log.gz 2024-09-18T01:18:58.993 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126446.log.gz 2024-09-18T01:18:58.993 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77347.log.gz 2024-09-18T01:18:58.993 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42394.log.gz 2024-09-18T01:18:58.993 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113704.log.gz 2024-09-18T01:18:58.993 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.41958.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114842.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101184.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69078.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115054.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33939.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.63656.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 20.5% -- replaced with /var/log/ceph/cluster1-client.admin.110103.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43355.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.49116.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.127669.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.64437.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.31489.log.gz 2024-09-18T01:18:58.994 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91762.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.39068.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88041.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53243.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.126151.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124051.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.83875.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48619.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113705.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.63575.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123503.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.106107.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92452.log.gz 2024-09-18T01:18:58.995 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39386.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122849.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.43844.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59081.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114548.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79604.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.122639.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124973.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77580.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104681.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.59500.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.33982.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.130238.log.gz 2024-09-18T01:18:58.996 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.76274.log.gz 2024-09-18T01:18:59.009 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34192.log.gz 2024-09-18T01:18:59.009 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68102.log.gz 2024-09-18T01:18:59.009 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88749.log.gz 2024-09-18T01:18:59.009 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68610.log.gz 2024-09-18T01:18:59.009 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54570.log.gz 2024-09-18T01:18:59.009 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124260.log.gz 2024-09-18T01:18:59.009 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.98282.log.gz 2024-09-18T01:18:59.009 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.97337.log.gz 2024-09-18T01:18:59.009 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.95960.log.gz 2024-09-18T01:18:59.009 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.69455.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91065.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123926.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.83397.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114127.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.75586.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.84300.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.109584.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.60329.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80154.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.49740.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.110697.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.47630.log.gz 2024-09-18T01:18:59.010 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.92043.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.54615.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.62670.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49895.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54806.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.73027.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78545.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88002.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.95904.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.52045.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39059.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.119678.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115683.log.gz 2024-09-18T01:18:59.011 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.38255.log.gz 2024-09-18T01:18:59.012 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111849.log.gz 2024-09-18T01:18:59.012 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44370.log.gz 2024-09-18T01:18:59.012 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68205.log.gz 2024-09-18T01:18:59.012 INFO:teuthology.orchestra.run.smithi049.stderr: 90.8% 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.40900.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.122368.log.gz 2024-09-18T01:18:59.012 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.012 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55540.log.gz 2024-09-18T01:18:59.012 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster1-client.admin.75840.log.gz 2024-09-18T01:18:59.012 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.105075.log.gz 2024-09-18T01:18:59.012 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55794.log.gz 2024-09-18T01:18:59.012 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122789.log.gz 2024-09-18T01:18:59.012 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.50419.log.gz 2024-09-18T01:18:59.012 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.72031.log.gz 2024-09-18T01:18:59.013 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.59290.log.gz 2024-09-18T01:18:59.013 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104384.log.gz 2024-09-18T01:18:59.013 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110781.log.gz 2024-09-18T01:18:59.013 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.104427.log.gz 2024-09-18T01:18:59.013 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40490.log.gz 2024-09-18T01:18:59.013 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33269.log.gz 2024-09-18T01:18:59.013 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.52830.log.gz 89.9% 2024-09-18T01:18:59.013 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.123984.log.gz 2024-09-18T01:18:59.013 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69622.log.gz 2024-09-18T01:18:59.014 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107732.log.gz 2024-09-18T01:18:59.014 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86158.log.gz 2024-09-18T01:18:59.014 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.86464.log.gz 2024-09-18T01:18:59.014 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53707.log.gz 2024-09-18T01:18:59.014 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.53176.log.gz 2024-09-18T01:18:59.014 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44203.log.gz 2024-09-18T01:18:59.014 INFO:teuthology.orchestra.run.smithi049.stderr: 86.5% -- replaced with /var/log/ceph/cluster1-client.admin.107451.log.gz 2024-09-18T01:18:59.014 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59941.log.gz 2024-09-18T01:18:59.014 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66098.log.gz 2024-09-18T01:18:59.014 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.138558.log.gz 2024-09-18T01:18:59.014 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.38253.log.gz 2024-09-18T01:18:59.014 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.97803.log.gz 2024-09-18T01:18:59.015 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.91405.log.gz 2024-09-18T01:18:59.015 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58744.log.gz 2024-09-18T01:18:59.015 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69242.log.gz 2024-09-18T01:18:59.015 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58955.log.gz 2024-09-18T01:18:59.015 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.125942.log.gz 2024-09-18T01:18:59.015 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36422.log.gz 2024-09-18T01:18:59.015 INFO:teuthology.orchestra.run.smithi049.stderr: 18.2% -- replaced with /var/log/ceph/cluster2-client.admin.83721.log.gz 2024-09-18T01:18:59.015 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41826.log.gz 2024-09-18T01:18:59.015 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114212.log.gz 2024-09-18T01:18:59.015 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.51361.log.gz 2024-09-18T01:18:59.015 INFO:teuthology.orchestra.run.smithi049.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-client.admin.130379.log.gz 2024-09-18T01:18:59.015 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104216.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99628.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.113322.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.74365.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100091.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56510.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36528.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112315.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59499.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37053.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.107659.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32317.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70631.log.gz 2024-09-18T01:18:59.016 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.98329.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79008.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.94057.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36252.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.72441.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57452.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78168.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123796.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46851.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93572.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92245.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 19.0% -- replaced with /var/log/ceph/cluster2-client.mirror.0.29998.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132982.log.gz 2024-09-18T01:18:59.017 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112904.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135136.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137035.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.72468.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41359.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.104552.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31446.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78965.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.113223.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.58449.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113650.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122427.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87201.log.gz 2024-09-18T01:18:59.018 INFO:teuthology.orchestra.run.smithi049.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51984.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113642.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123440.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137450.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.117974.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.49280.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.134880.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.128586.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37577.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.110024.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.140460.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.83951.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92330.log.gz 2024-09-18T01:18:59.019 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.131314.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.49092.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.49822.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.126866.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71359.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88874.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.127842.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90918.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.117295.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80489.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.133196.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.36152.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.120084.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster1-client.admin.109893.log.gz 2024-09-18T01:18:59.020 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111264.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.109620.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134038.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101015.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.102999.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112693.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.120180.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.135473.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.98485.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82000.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.105105.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62166.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124109.log.gz 2024-09-18T01:18:59.021 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76730.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.40873.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134927.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.87509.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136168.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44262.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121301.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68399.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.30779.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34844.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122376.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93549.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45564.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118631.log.gz 2024-09-18T01:18:59.022 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76947.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101018.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87652.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42834.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81625.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.129562.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.74734.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112019.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55478.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.125521.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100092.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 80.5% -- replaced with /var/log/ceph/cluster1-client.mirror.76265.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.95962.log.gz 2024-09-18T01:18:59.023 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89864.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112733.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125874.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91129.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% 89.9% -- replaced with /var/log/ceph/cluster2-client.admin.117378.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.78336.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101898.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91193.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.118367.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123294.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.128753.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96365.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35792.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104469.log.gz 2024-09-18T01:18:59.024 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135091.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.84736.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 66.3% -- replaced with /var/log/ceph/cluster1-client.mirror.63334.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34150.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136834.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104512.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99756.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.87400.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 2.3% -- replaced with /var/log/ceph/cluster2-client.mirror.29874.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.96292.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.130519.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.32113.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56719.log.gz 2024-09-18T01:18:59.025 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132514.log.gz 2024-09-18T01:18:59.026 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110740.log.gz 2024-09-18T01:18:59.026 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135344.log.gz 2024-09-18T01:18:59.026 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.119041.log.gz 2024-09-18T01:18:59.026 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102233.log.gz 2024-09-18T01:18:59.026 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35096.log.gz 2024-09-18T01:18:59.026 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100386.log.gz 2024-09-18T01:18:59.026 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.131111.log.gz 2024-09-18T01:18:59.026 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75241.log.gz 2024-09-18T01:18:59.026 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35623.log.gz 2024-09-18T01:18:59.026 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133828.log.gz 2024-09-18T01:18:59.026 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47144.log.gz 2024-09-18T01:18:59.026 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137245.log.gz 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.108363.log.gz 89.6% 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.129161.log.gz 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126023.log.gz 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.109767.log.gz 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.94516.log.gz 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.110627.log.gz 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47738.log.gz 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.41128.log.gz 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.33940.log.gz 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.50109.log.gz 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47485.log.gz 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.120886.log.gz 2024-09-18T01:18:59.027 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46513.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster1-client.admin.120955.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111326.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.48863.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54721.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46574.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.74932.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91613.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115009.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104659.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44790.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.120351.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112314.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89026.log.gz 2024-09-18T01:18:59.028 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134208.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.90983.log.gz 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81790.log.gz 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.132559.log.gz 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81664.log.gz 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66759.log.gz 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96479.log.gz 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100805.log.gz 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85323.log.gz 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48052.log.gz 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59162.log.gz 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.67725.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.73255.log.gz 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.125276.log.gz 2024-09-18T01:18:59.029 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69666.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.94808.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.72504.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.64057.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.admin.29584.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77834.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39064.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34466.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112229.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127054.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.38631.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79645.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.138638.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93086.log.gz 2024-09-18T01:18:59.030 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126572.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133911.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54092.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44514.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.64280.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.69662.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112586.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69368.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.112947.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135303.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124218.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.104426.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44304.log.gz 2024-09-18T01:18:59.031 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster2-client.admin.73557.log.gz 2024-09-18T01:18:59.032 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69623.log.gz 89.8% 2024-09-18T01:18:59.032 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.113364.log.gz 2024-09-18T01:18:59.032 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.96198.log.gz 2024-09-18T01:18:59.032 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.60614.log.gz 2024-09-18T01:18:59.032 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122999.log.gz 2024-09-18T01:18:59.032 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.31396.log.gz 2024-09-18T01:18:59.032 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster1-client.admin.87713.log.gz 2024-09-18T01:18:59.032 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster2-client.admin.61977.log.gz 2024-09-18T01:18:59.032 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster1-client.admin.31552.log.gz 2024-09-18T01:18:59.032 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.63842.log.gz 2024-09-18T01:18:59.032 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46617.log.gz 2024-09-18T01:18:59.032 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.96312.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90558.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85417.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53872.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.90010.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.104988.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55058.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133618.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.42510.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.125686.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.139726.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.126369.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.44329.log.gz 2024-09-18T01:18:59.033 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.85136.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58155.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 58.3% -- replaced with /var/log/ceph/cluster1-client.mirror.29703.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65497.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134840.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36991.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126526.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59139.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.119611.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67472.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53962.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123001.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.95524.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.94809.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101941.log.gz 2024-09-18T01:18:59.034 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.44957.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125605.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90302.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100009.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89925.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113157.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.119944.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102363.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.98954.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92665.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.76316.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90284.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42878.log.gz 2024-09-18T01:18:59.035 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.138513.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31902.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113918.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43191.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99714.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55226.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.106214.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92957.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77579.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70522.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70418.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.87442.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.52539.log.gz 2024-09-18T01:18:59.036 INFO:teuthology.orchestra.run.smithi049.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.120820.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80320.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59878.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103797.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65044.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.114633.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.75202.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.95263.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43192.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.135789.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34004.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78713.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster2-client.admin.39421.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137244.log.gz 2024-09-18T01:18:59.037 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100595.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75734.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80783.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 91.0% -- replaced with /var/log/ceph/cluster1-client.admin.85619.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65626.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123420.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135556.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90767.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111110.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster1-client.admin.52486.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.133910.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58869.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45734.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102873.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68439.log.gz 2024-09-18T01:18:59.038 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70672.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.108780.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65759.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.135345.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.85820.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.139427.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122665.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56257.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79693.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.139986.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99587.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45103.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.62697.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89655.log.gz 2024-09-18T01:18:59.039 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.120152.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36108.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.71864.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112734.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78674.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34686.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.99327.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.78422.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102193.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.131993.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53348.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79480.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.65347.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.86900.log.gz 2024-09-18T01:18:59.040 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36295.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45882.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66675.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92035.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136111.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.57689.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.129949.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88061.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112376.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110656.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.129835.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92602.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60161.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54361.log.gz 2024-09-18T01:18:59.041 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39067.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111958.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68145.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.62328.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.97577.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58405.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.55690.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49940.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124949.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91192.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115056.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133448.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79646.log.gz 2024-09-18T01:18:59.042 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.120884.log.gz 2024-09-18T01:18:59.043 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.105790.log.gz 2024-09-18T01:18:59.043 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.116323.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.32537.log.gz 2024-09-18T01:18:59.043 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.043 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122098.log.gz 2024-09-18T01:18:59.043 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.126107.log.gz 2024-09-18T01:18:59.043 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90496.log.gz 2024-09-18T01:18:59.043 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44410.log.gz 2024-09-18T01:18:59.043 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.116816.log.gz 2024-09-18T01:18:59.043 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.61603.log.gz 2024-09-18T01:18:59.043 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.75976.log.gz 2024-09-18T01:18:59.043 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88604.log.gz 2024-09-18T01:18:59.048 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122873.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.110199.log.gz 2024-09-18T01:18:59.048 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.048 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44750.log.gz 2024-09-18T01:18:59.048 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79690.log.gz 2024-09-18T01:18:59.048 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91970.log.gz 2024-09-18T01:18:59.048 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59246.log.gz 2024-09-18T01:18:59.048 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.41101.log.gz 2024-09-18T01:18:59.048 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.124133.log.gz 2024-09-18T01:18:59.048 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102785.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.126000.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45001.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136781.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47739.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47696.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59080.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48116.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.84121.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90495.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.72143.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91973.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44791.log.gz 2024-09-18T01:18:59.049 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134287.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.105470.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.mirror.121221.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.69369.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52098.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123269.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.119497.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102447.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122302.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99246.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59036.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.32997.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.49218.log.gz 2024-09-18T01:18:59.050 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100554.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.63484.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.61036.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121782.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58198.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91486.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79734.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67177.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.74392.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.119181.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137241.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.120822.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.64019.log.gz 2024-09-18T01:18:59.051 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.105708.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88538.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56638.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67051.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133658.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99586.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47361.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115326.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.39693.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.129896.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.83381.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87003.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.119155.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36927.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58929.log.gz 2024-09-18T01:18:59.052 INFO:teuthology.orchestra.run.smithi049.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53666.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45630.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90075.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112692.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.83580.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.114882.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88606.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88958.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.113578.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57517.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65498.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.68948.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.95538.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.134670.log.gz 2024-09-18T01:18:59.053 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.36361.log.gz 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.126152.log.gz 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.114066.log.gz 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123419.log.gz 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.118900.log.gz 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82360.log.gz 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102700.log.gz 89.9% 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.91763.log.gz 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.140013.log.gz 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.92812.log.gz 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.107337.log.gz 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102492.log.gz 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39045.log.gz 2024-09-18T01:18:59.054 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.76088.log.gz 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.106215.log.gz 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.82654.log.gz 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.91342.log.gz 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41590.log.gz 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115335.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.45732.log.gz 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89442.log.gz 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.138602.log.gz 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87508.log.gz 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.135770.log.gz 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.95129.log.gz 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.98419.log.gz 2024-09-18T01:18:59.055 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44117.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77924.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90345.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.106176.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55603.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55479.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78378.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45986.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125309.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93295.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47569.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89929.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.119809.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75627.log.gz 2024-09-18T01:18:59.056 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42981.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43134.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80488.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90768.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45421.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.41743.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.41518.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.63467.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133497.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.31131.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.32786.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.119356.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63179.log.gz 2024-09-18T01:18:59.057 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90497.log.gz 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36632.log.gz 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35519.log.gz 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45378.log.gz 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.52072.log.gz 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.97778.log.gz 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124006.log.gz 18.8% 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.30667.log.gz 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92580.log.gz 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115475.log.gz 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70125.log.gz 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.138834.log.gz 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43489.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.78504.log.gz 2024-09-18T01:18:59.058 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123209.log.gz 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68779.log.gz 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.87141.log.gz 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.90917.log.gz 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45144.log.gz 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104177.log.gz 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44472.log.gz 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54764.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.90641.log.gz 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101730.log.gz 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.30644.log.gz 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.98051.log.gz 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45460.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.125475.log.gz 2024-09-18T01:18:59.059 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.51133.log.gz 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115390.log.gz 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.139890.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.106283.log.gz 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.85163.log.gz 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52759.log.gz 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.98637.log.gz 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.50893.log.gz 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67306.log.gz 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124975.log.gz 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.127721.log.gz 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77130.log.gz 2024-09-18T01:18:59.060 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127306.log.gz 2024-09-18T01:18:59.061 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66273.log.gz 2024-09-18T01:18:59.061 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.42310.log.gz 2024-09-18T01:18:59.061 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104616.log.gz 2024-09-18T01:18:59.061 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.49959.log.gz 2024-09-18T01:18:59.061 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41693.log.gz 2024-09-18T01:18:59.061 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.86157.log.gz 2024-09-18T01:18:59.061 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68736.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.133175.log.gz 2024-09-18T01:18:59.061 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.061 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.107545.log.gz 2024-09-18T01:18:59.061 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.99389.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.73530.log.gz 2024-09-18T01:18:59.061 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.061 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91403.log.gz 2024-09-18T01:18:59.064 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.135346.log.gz 2024-09-18T01:18:59.064 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76567.log.gz 2024-09-18T01:18:59.064 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.30366.log.gz 2024-09-18T01:18:59.064 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.62837.log.gz 2024-09-18T01:18:59.064 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.106281.log.gz 2024-09-18T01:18:59.065 INFO:teuthology.orchestra.run.smithi049.stderr: 67.3% -- replaced with /var/log/ceph/cluster1-client.mirror.74459.log.gz 2024-09-18T01:18:59.065 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.97482.log.gz 2024-09-18T01:18:59.065 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137410.log.gz 2024-09-18T01:18:59.065 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104785.log.gz 2024-09-18T01:18:59.065 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42876.log.gz 2024-09-18T01:18:59.065 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.95573.log.gz 2024-09-18T01:18:59.065 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.121065.log.gz 2024-09-18T01:18:59.065 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138126.log.gz 2024-09-18T01:18:59.065 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.44371.log.gz 2024-09-18T01:18:59.065 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.110269.log.gz 2024-09-18T01:18:59.065 INFO:teuthology.orchestra.run.smithi049.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.41874.log.gz 2024-09-18T01:18:59.065 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster1-client.admin.108803.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82001.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96935.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113112.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135729.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80869.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34845.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.38798.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66190.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87306.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.74003.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133279.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90771.log.gz 2024-09-18T01:18:59.066 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85987.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster2-client.admin.140687.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125244.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.66716.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72986.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101099.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.96681.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.74801.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43464.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34529.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133576.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.104176.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.39983.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.97823.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.37862.log.gz 2024-09-18T01:18:59.067 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70186.log.gz 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 67.3% -- replaced with /var/log/ceph/cluster1-client.mirror.73889.log.gz 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.61354.log.gz 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101564.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.125318.log.gz 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.132646.log.gz 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.63745.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.99815.log.gz 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.61228.log.gz 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135690.log.gz 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33418.log.gz 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123925.log.gz 2024-09-18T01:18:59.068 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.137371.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92160.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57559.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47105.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39054.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.36107.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82045.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 68.0% -- replaced with /var/log/ceph/cluster1-client.mirror.130788.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.41321.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.128103.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44263.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46156.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43697.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54269.log.gz 2024-09-18T01:18:59.069 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101939.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83465.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61823.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101774.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81497.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.81160.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65844.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79260.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57349.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133867.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135261.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.87021.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74418.log.gz 2024-09-18T01:18:59.070 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103123.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.98457.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136824.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53489.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.60160.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91132.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123713.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75376.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57518.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 80.5% -- replaced with /var/log/ceph/cluster1-client.mirror.108718.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82127.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.32114.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.84636.log.gz 2024-09-18T01:18:59.071 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 18.2% -- replaced with /var/log/ceph/cluster1-client.mirror.112020.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.65207.log.gz 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67137.log.gz 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32046.log.gz 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.60877.log.gz 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99500.log.gz 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.36042.log.gz 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115746.log.gz 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.64619.log.gz 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.116283.log.gz 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.60447.log.gz 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35242.log.gz 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49632.log.gz 2024-09-18T01:18:59.072 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133550.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.71680.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62536.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45673.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46093.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102957.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.131954.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41539.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.102659.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125034.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.106661.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90138.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 68.3% -- replaced with /var/log/ceph/cluster1-client.mirror.97090.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122577.log.gz 2024-09-18T01:18:59.073 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102108.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: 3.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29891.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: 90.7% -- replaced with /var/log/ceph/cluster1-client.admin.130493.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.32421.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.57393.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.71225.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136190.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55015.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79434.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% 89.9% -- replaced with /var/log/ceph/cluster2-client.admin.38800.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.124048.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48411.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.106603.log.gz 2024-09-18T01:18:59.074 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41626.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.82934.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87853.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127201.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46009.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133909.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.44621.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35877.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35662.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137031.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115829.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81456.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112059.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31900.log.gz 2024-09-18T01:18:59.075 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.135134.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.76512.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.116261.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70713.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79858.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70252.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32679.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58574.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.87051.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 67.8% -- replaced with /var/log/ceph/cluster1-client.mirror.108229.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55923.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.105869.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.65969.log.gz 2024-09-18T01:18:59.076 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.109977.log.gz 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52175.log.gz 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34047.log.gz 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66758.log.gz 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89717.log.gz 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65182.log.gz 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126422.log.gz 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75561.log.gz 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41960.log.gz 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36002.log.gz 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.admin.107820.log.gz 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122016.log.gz 2024-09-18T01:18:59.077 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.98933.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125812.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86774.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54931.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90009.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 57.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29857.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.admin.130151.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134248.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.50339.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102362.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.74593.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71424.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77244.log.gz 2024-09-18T01:18:59.078 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52713.log.gz 2024-09-18T01:18:59.079 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113321.log.gz 2024-09-18T01:18:59.079 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.109734.log.gz 2024-09-18T01:18:59.079 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114170.log.gz 2024-09-18T01:18:59.079 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.83081.log.gz 2024-09-18T01:18:59.083 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.admin.62114.log.gz 2024-09-18T01:18:59.084 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114382.log.gz 2024-09-18T01:18:59.084 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78253.log.gz 2024-09-18T01:18:59.084 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster2-client.admin.73358.log.gz 2024-09-18T01:18:59.084 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65108.log.gz 2024-09-18T01:18:59.084 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101309.log.gz 2024-09-18T01:18:59.084 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77108.log.gz 2024-09-18T01:18:59.084 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133405.log.gz 2024-09-18T01:18:59.084 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121702.log.gz 2024-09-18T01:18:59.084 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37863.log.gz 2024-09-18T01:18:59.084 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.127427.log.gz 2024-09-18T01:18:59.084 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.107125.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57308.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 90.9% -- replaced with /var/log/ceph/cluster1-client.admin.97023.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.32789.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.122872.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.102658.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125184.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56048.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31825.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92662.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45312.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91554.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137792.log.gz 2024-09-18T01:18:59.085 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53306.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.139002.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75414.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102066.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111892.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.32174.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44055.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.49595.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43843.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.74231.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136404.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.128581.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 18.2% -- replaced with /var/log/ceph/cluster1-client.admin.53371.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.72273.log.gz 2024-09-18T01:18:59.086 INFO:teuthology.orchestra.run.smithi049.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.62469.log.gz 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33795.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110698.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79051.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98837.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137581.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63085.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100724.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96166.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93024.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134037.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44327.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106957.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105986.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69494.log 2024-09-18T01:18:59.087 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78884.log 2024-09-18T01:18:59.088 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112272.log 2024-09-18T01:18:59.089 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126947.log 2024-09-18T01:18:59.089 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133575.log 2024-09-18T01:18:59.089 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116658.log 2024-09-18T01:18:59.089 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62204.log 2024-09-18T01:18:59.089 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55141.log 2024-09-18T01:18:59.090 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52484.log 2024-09-18T01:18:59.090 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71936.log 2024-09-18T01:18:59.090 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101438.log 2024-09-18T01:18:59.090 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86479.log 2024-09-18T01:18:59.091 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121004.log 2024-09-18T01:18:59.091 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62723.log 2024-09-18T01:18:59.091 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81580.log 2024-09-18T01:18:59.091 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120462.log 2024-09-18T01:18:59.092 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101773.log 2024-09-18T01:18:59.092 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67659.log 2024-09-18T01:18:59.092 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97893.log 2024-09-18T01:18:59.092 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50587.log 2024-09-18T01:18:59.093 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89085.log 2024-09-18T01:18:59.093 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117732.log 2024-09-18T01:18:59.093 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142416.log 2024-09-18T01:18:59.093 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56761.log 2024-09-18T01:18:59.094 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79346.log 2024-09-18T01:18:59.094 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111640.log 2024-09-18T01:18:59.094 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64546.log 2024-09-18T01:18:59.095 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63878.log 2024-09-18T01:18:59.095 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70927.log 2024-09-18T01:18:59.095 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54182.log 2024-09-18T01:18:59.096 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81204.log 2024-09-18T01:18:59.096 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29754.log 2024-09-18T01:18:59.096 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59077.log 2024-09-18T01:18:59.096 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137033.log 2024-09-18T01:18:59.097 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119873.log 2024-09-18T01:18:59.097 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30676.log 2024-09-18T01:18:59.097 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77707.log 2024-09-18T01:18:59.097 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29096.log 2024-09-18T01:18:59.098 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127556.log 2024-09-18T01:18:59.098 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66443.log 2024-09-18T01:18:59.098 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28742.log 2024-09-18T01:18:59.099 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70127.log 2024-09-18T01:18:59.099 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41195.log 2024-09-18T01:18:59.100 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85277.log 2024-09-18T01:18:59.100 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121728.log 2024-09-18T01:18:59.100 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59791.log 2024-09-18T01:18:59.104 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120286.log 2024-09-18T01:18:59.115 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62764.log 2024-09-18T01:18:59.115 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121085.log 2024-09-18T01:18:59.115 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102023.log 2024-09-18T01:18:59.115 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.118699.log 2024-09-18T01:18:59.115 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40967.log 2024-09-18T01:18:59.115 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108302.log 2024-09-18T01:18:59.115 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107980.log 2024-09-18T01:18:59.115 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139430.log 2024-09-18T01:18:59.115 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105223.log 2024-09-18T01:18:59.115 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59456.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89237.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115117.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133155.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129471.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72120.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116258.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141079.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87753.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34049.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79563.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137034.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64763.log 2024-09-18T01:18:59.116 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42319.log 2024-09-18T01:18:59.117 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32069.log 2024-09-18T01:18:59.117 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82935.log 2024-09-18T01:18:59.117 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113010.log 2024-09-18T01:18:59.117 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86236.log 2024-09-18T01:18:59.117 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30580.log 2024-09-18T01:18:59.117 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102742.log 2024-09-18T01:18:59.118 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90348.log 2024-09-18T01:18:59.118 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51455.log 2024-09-18T01:18:59.119 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132750.log 2024-09-18T01:18:59.119 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105479.log 2024-09-18T01:18:59.119 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93356.log 2024-09-18T01:18:59.119 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128179.log 2024-09-18T01:18:59.119 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59711.log 2024-09-18T01:18:59.119 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69918.log 2024-09-18T01:18:59.119 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90896.log 2024-09-18T01:18:59.119 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130265.log 2024-09-18T01:18:59.119 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49803.log 2024-09-18T01:18:59.119 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.107364.log 2024-09-18T01:18:59.119 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111161.log 2024-09-18T01:18:59.120 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85458.log 2024-09-18T01:18:59.121 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140480.log 2024-09-18T01:18:59.121 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134139.log 2024-09-18T01:18:59.121 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91319.log 2024-09-18T01:18:59.121 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140193.log 2024-09-18T01:18:59.121 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57732.log 2024-09-18T01:18:59.121 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34635.log 2024-09-18T01:18:59.122 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89801.log 2024-09-18T01:18:59.122 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53488.log 2024-09-18T01:18:59.122 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67305.log 2024-09-18T01:18:59.122 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81037.log 2024-09-18T01:18:59.122 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126316.log 2024-09-18T01:18:59.123 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62971.log 2024-09-18T01:18:59.123 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118180.log 2024-09-18T01:18:59.123 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83291.log 2024-09-18T01:18:59.123 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129259.log 2024-09-18T01:18:59.124 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110405.log 2024-09-18T01:18:59.124 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123924.log 2024-09-18T01:18:59.124 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96955.log 2024-09-18T01:18:59.124 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67347.log 2024-09-18T01:18:59.124 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129085.log 2024-09-18T01:18:59.125 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42265.log 2024-09-18T01:18:59.125 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58995.log 2024-09-18T01:18:59.125 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37262.log 2024-09-18T01:18:59.125 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102043.log 2024-09-18T01:18:59.126 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99839.log 2024-09-18T01:18:59.126 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51434.log 2024-09-18T01:18:59.126 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64859.log 2024-09-18T01:18:59.127 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123292.log 2024-09-18T01:18:59.127 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.132223.log 2024-09-18T01:18:59.127 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71583.log 2024-09-18T01:18:59.127 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96178.log 2024-09-18T01:18:59.128 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53618.log 2024-09-18T01:18:59.132 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127286.log 2024-09-18T01:18:59.132 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135090.log 2024-09-18T01:18:59.132 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25267.log 2024-09-18T01:18:59.132 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111076.log 2024-09-18T01:18:59.132 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104006.log 2024-09-18T01:18:59.132 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129762.log 2024-09-18T01:18:59.132 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90852.log 2024-09-18T01:18:59.132 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51931.log 2024-09-18T01:18:59.133 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105097.log 2024-09-18T01:18:59.133 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62314.log 2024-09-18T01:18:59.133 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75895.log 2024-09-18T01:18:59.134 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120699.log 2024-09-18T01:18:59.134 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71837.log 2024-09-18T01:18:59.134 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135984.log 2024-09-18T01:18:59.134 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121543.log 2024-09-18T01:18:59.134 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62245.log 2024-09-18T01:18:59.135 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65714.log 2024-09-18T01:18:59.135 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89632.log 2024-09-18T01:18:59.135 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102830.log 2024-09-18T01:18:59.135 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47486.log 2024-09-18T01:18:59.135 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51589.log 2024-09-18T01:18:59.136 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128491.log 2024-09-18T01:18:59.136 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128710.log 2024-09-18T01:18:59.136 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101604.log 2024-09-18T01:18:59.136 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112649.log 2024-09-18T01:18:59.137 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98525.log 2024-09-18T01:18:59.137 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96225.log 2024-09-18T01:18:59.137 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140350.log 2024-09-18T01:18:59.138 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113366.log 2024-09-18T01:18:59.138 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101646.log 2024-09-18T01:18:59.138 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118953.log 2024-09-18T01:18:59.138 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49283.log 2024-09-18T01:18:59.139 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102278.log 2024-09-18T01:18:59.139 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71246.log 2024-09-18T01:18:59.139 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129876.log 2024-09-18T01:18:59.139 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50902.log 2024-09-18T01:18:59.140 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54486.log 2024-09-18T01:18:59.140 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117388.log 2024-09-18T01:18:59.140 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112377.log 2024-09-18T01:18:59.141 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98998.log 2024-09-18T01:18:59.141 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91907.log 2024-09-18T01:18:59.141 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.129695.log 2024-09-18T01:18:59.142 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82556.log 2024-09-18T01:18:59.142 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71034.log 2024-09-18T01:18:59.142 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65542.log 2024-09-18T01:18:59.142 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118314.log 2024-09-18T01:18:59.142 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95952.log 2024-09-18T01:18:59.142 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113155.log 2024-09-18T01:18:59.144 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30476.log 2024-09-18T01:18:59.144 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73677.log 2024-09-18T01:18:59.144 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75650.log 2024-09-18T01:18:59.144 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136821.log 2024-09-18T01:18:59.144 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56888.log 2024-09-18T01:18:59.146 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84035.log 2024-09-18T01:18:59.146 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109176.log 2024-09-18T01:18:59.146 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84501.log 2024-09-18T01:18:59.146 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75522.log 2024-09-18T01:18:59.146 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74930.log 2024-09-18T01:18:59.146 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61915.log 2024-09-18T01:18:59.146 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128713.log 2024-09-18T01:18:59.146 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133365.log 2024-09-18T01:18:59.147 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141334.log 2024-09-18T01:18:59.147 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74278.log 2024-09-18T01:18:59.147 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128711.log 2024-09-18T01:18:59.147 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63536.log 2024-09-18T01:18:59.147 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130952.log 2024-09-18T01:18:59.148 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79217.log 2024-09-18T01:18:59.149 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102234.log 2024-09-18T01:18:59.149 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.118088.log 2024-09-18T01:18:59.149 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119269.log 2024-09-18T01:18:59.149 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111476.log 2024-09-18T01:18:59.149 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62811.log 2024-09-18T01:18:59.149 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.95128.log 2024-09-18T01:18:59.150 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107301.log 2024-09-18T01:18:59.150 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60110.log 2024-09-18T01:18:59.150 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84935.log 2024-09-18T01:18:59.150 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57265.log 2024-09-18T01:18:59.150 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32464.log 2024-09-18T01:18:59.151 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46094.log 2024-09-18T01:18:59.151 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72578.log 2024-09-18T01:18:59.151 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142182.log 2024-09-18T01:18:59.152 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104218.log 2024-09-18T01:18:59.152 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138252.log 2024-09-18T01:18:59.152 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96748.log 2024-09-18T01:18:59.152 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38246.log 2024-09-18T01:18:59.153 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81748.log 2024-09-18T01:18:59.153 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42918.log 2024-09-18T01:18:59.153 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132980.log 2024-09-18T01:18:59.154 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114906.log 2024-09-18T01:18:59.154 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135940.log 2024-09-18T01:18:59.154 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87279.log 2024-09-18T01:18:59.154 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31750.log 2024-09-18T01:18:59.154 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129395.log 2024-09-18T01:18:59.155 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81244.log 2024-09-18T01:18:59.155 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117991.log 2024-09-18T01:18:59.156 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63039.log 2024-09-18T01:18:59.156 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139467.log 2024-09-18T01:18:59.156 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77414.log 2024-09-18T01:18:59.156 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123502.log 2024-09-18T01:18:59.157 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126736.log 2024-09-18T01:18:59.157 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33206.log 2024-09-18T01:18:59.157 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141448.log 2024-09-18T01:18:59.157 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131134.log 2024-09-18T01:18:59.157 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118434.log 2024-09-18T01:18:59.157 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37135.log 2024-09-18T01:18:59.158 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95236.log 2024-09-18T01:18:59.159 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114760.log 2024-09-18T01:18:59.159 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71293.log 2024-09-18T01:18:59.159 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64207.log 2024-09-18T01:18:59.159 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113184.log 2024-09-18T01:18:59.160 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116349.log 2024-09-18T01:18:59.160 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77046.log 2024-09-18T01:18:59.160 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109850.log 2024-09-18T01:18:59.161 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42696.log 2024-09-18T01:18:59.161 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121635.log 2024-09-18T01:18:59.161 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109512.log 2024-09-18T01:18:59.161 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141676.log 2024-09-18T01:18:59.165 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64502.log 2024-09-18T01:18:59.165 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118341.log 2024-09-18T01:18:59.165 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40030.log 2024-09-18T01:18:59.165 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106174.log 2024-09-18T01:18:59.165 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91697.log 2024-09-18T01:18:59.165 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112798.log 2024-09-18T01:18:59.165 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66842.log 2024-09-18T01:18:59.165 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139086.log 2024-09-18T01:18:59.165 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132385.log 2024-09-18T01:18:59.166 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108966.log 2024-09-18T01:18:59.166 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134140.log 2024-09-18T01:18:59.167 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43698.log 2024-09-18T01:18:59.167 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75374.log 2024-09-18T01:18:59.167 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76320.log 2024-09-18T01:18:59.167 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76814.log 2024-09-18T01:18:59.167 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134924.log 2024-09-18T01:18:59.167 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66631.log 2024-09-18T01:18:59.167 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79136.log 2024-09-18T01:18:59.167 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118528.log 2024-09-18T01:18:59.168 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88372.log 2024-09-18T01:18:59.169 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125454.log 2024-09-18T01:18:59.169 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85551.log 2024-09-18T01:18:59.169 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114908.log 2024-09-18T01:18:59.169 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94806.log 2024-09-18T01:18:59.169 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110990.log 2024-09-18T01:18:59.170 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90714.log 2024-09-18T01:18:59.170 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55268.log 2024-09-18T01:18:59.170 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70253.log 2024-09-18T01:18:59.170 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46154.log 2024-09-18T01:18:59.170 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137455.log 2024-09-18T01:18:59.171 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35684.log 2024-09-18T01:18:59.171 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50318.log 2024-09-18T01:18:59.171 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125685.log 2024-09-18T01:18:59.171 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135135.log 2024-09-18T01:18:59.171 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118514.log 2024-09-18T01:18:59.172 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138930.log 2024-09-18T01:18:59.172 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57097.log 2024-09-18T01:18:59.172 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.107793.log 2024-09-18T01:18:59.172 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28534.log 2024-09-18T01:18:59.173 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134296.log 2024-09-18T01:18:59.173 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101143.log 2024-09-18T01:18:59.173 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62878.log 2024-09-18T01:18:59.173 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110865.log 2024-09-18T01:18:59.174 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49409.log 2024-09-18T01:18:59.174 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81876.log 2024-09-18T01:18:59.174 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73104.log 2024-09-18T01:18:59.175 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141287.log 2024-09-18T01:18:59.175 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37202.log 2024-09-18T01:18:59.175 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111749.log 2024-09-18T01:18:59.175 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44747.log 2024-09-18T01:18:59.175 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79776.log 2024-09-18T01:18:59.175 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73653.log 2024-09-18T01:18:59.176 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33520.log 2024-09-18T01:18:59.176 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139337.log 2024-09-18T01:18:59.177 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63153.log 2024-09-18T01:18:59.177 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93807.log 2024-09-18T01:18:59.177 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93210.log 2024-09-18T01:18:59.178 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66444.log 2024-09-18T01:18:59.178 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53616.log 2024-09-18T01:18:59.178 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111389.log 2024-09-18T01:18:59.178 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61630.log 2024-09-18T01:18:59.178 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107504.log 2024-09-18T01:18:59.179 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90430.log 2024-09-18T01:18:59.179 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55924.log 2024-09-18T01:18:59.179 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.107706.log 2024-09-18T01:18:59.180 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48302.log 2024-09-18T01:18:59.180 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101183.log 2024-09-18T01:18:59.180 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62297.log 2024-09-18T01:18:59.180 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33477.log 2024-09-18T01:18:59.180 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25336.log 2024-09-18T01:18:59.182 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51703.log 2024-09-18T01:18:59.182 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42002.log 2024-09-18T01:18:59.182 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105240.log 2024-09-18T01:18:59.182 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86194.log 2024-09-18T01:18:59.182 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62650.log 2024-09-18T01:18:59.183 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46053.log 2024-09-18T01:18:59.183 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30389.log 2024-09-18T01:18:59.183 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81205.log 2024-09-18T01:18:59.183 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49541.log 2024-09-18T01:18:59.183 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117153.log 2024-09-18T01:18:59.184 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73011.log 2024-09-18T01:18:59.184 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82782.log 2024-09-18T01:18:59.184 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74666.log 2024-09-18T01:18:59.184 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34740.log 2024-09-18T01:18:59.184 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39252.log 2024-09-18T01:18:59.184 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35159.log 2024-09-18T01:18:59.184 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90642.log 2024-09-18T01:18:59.186 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130580.log 2024-09-18T01:18:59.186 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132269.log 2024-09-18T01:18:59.186 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78630.log 2024-09-18T01:18:59.186 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82338.log 2024-09-18T01:18:59.186 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25419.log 2024-09-18T01:18:59.186 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64801.log 2024-09-18T01:18:59.186 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101877.log 2024-09-18T01:18:59.187 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48950.log 2024-09-18T01:18:59.187 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49399.log 2024-09-18T01:18:59.188 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74506.log 2024-09-18T01:18:59.188 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58871.log 2024-09-18T01:18:59.188 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95950.log 2024-09-18T01:18:59.188 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75796.log 2024-09-18T01:18:59.188 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131427.log 2024-09-18T01:18:59.189 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56554.log 2024-09-18T01:18:59.190 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86657.log 2024-09-18T01:18:59.190 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87980.log 2024-09-18T01:18:59.190 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101437.log 2024-09-18T01:18:59.190 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69706.log 2024-09-18T01:18:59.190 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139079.log 2024-09-18T01:18:59.190 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52345.log 2024-09-18T01:18:59.190 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106012.log 2024-09-18T01:18:59.190 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83969.log 2024-09-18T01:18:59.191 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50377.log 2024-09-18T01:18:59.191 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136611.log 2024-09-18T01:18:59.191 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116589.log 2024-09-18T01:18:59.192 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68926.log 2024-09-18T01:18:59.192 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59666.log 2024-09-18T01:18:59.192 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108115.log 2024-09-18T01:18:59.192 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132282.log 2024-09-18T01:18:59.193 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34783.log 2024-09-18T01:18:59.193 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69033.log 2024-09-18T01:18:59.193 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25487.log 2024-09-18T01:18:59.194 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137875.log 2024-09-18T01:18:59.194 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30542.log 2024-09-18T01:18:59.194 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62381.log 2024-09-18T01:18:59.194 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94518.log 2024-09-18T01:18:59.194 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113009.log 2024-09-18T01:18:59.195 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117456.log 2024-09-18T01:18:59.195 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88792.log 2024-09-18T01:18:59.195 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139358.log 2024-09-18T01:18:59.195 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31086.log 2024-09-18T01:18:59.195 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.107907.log 2024-09-18T01:18:59.195 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91276.log 2024-09-18T01:18:59.197 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59624.log 2024-09-18T01:18:59.197 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73566.log 2024-09-18T01:18:59.197 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39387.log 2024-09-18T01:18:59.197 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79522.log 2024-09-18T01:18:59.197 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121970.log 2024-09-18T01:18:59.197 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61278.log 2024-09-18T01:18:59.197 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98307.log 2024-09-18T01:18:59.199 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33480.log 2024-09-18T01:18:59.199 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61705.log 2024-09-18T01:18:59.199 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37822.log 2024-09-18T01:18:59.200 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111599.log 2024-09-18T01:18:59.200 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88582.log 2024-09-18T01:18:59.200 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62301.log 2024-09-18T01:18:59.200 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126654.log 2024-09-18T01:18:59.200 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121912.log 2024-09-18T01:18:59.200 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91402.log 2024-09-18T01:18:59.200 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136822.log 2024-09-18T01:18:59.200 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120212.log 2024-09-18T01:18:59.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100260.log 2024-09-18T01:18:59.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111751.log 2024-09-18T01:18:59.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33562.log 2024-09-18T01:18:59.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70334.log 2024-09-18T01:18:59.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129633.log 2024-09-18T01:18:59.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66399.log 2024-09-18T01:18:59.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69496.log 2024-09-18T01:18:59.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32514.log 2024-09-18T01:18:59.202 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122747.log 2024-09-18T01:18:59.204 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29346.log 2024-09-18T01:18:59.204 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42574.log 2024-09-18T01:18:59.204 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78546.log 2024-09-18T01:18:59.204 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134498.log 2024-09-18T01:18:59.204 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39570.log 2024-09-18T01:18:59.204 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100934.log 2024-09-18T01:18:59.204 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76791.log 2024-09-18T01:18:59.204 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31748.log 2024-09-18T01:18:59.206 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134458.log 2024-09-18T01:18:59.206 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34214.log 2024-09-18T01:18:59.206 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43272.log 2024-09-18T01:18:59.206 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50989.log 2024-09-18T01:18:59.206 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80155.log 2024-09-18T01:18:59.206 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66968.log 2024-09-18T01:18:59.206 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71096.log 2024-09-18T01:18:59.206 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61441.log 2024-09-18T01:18:59.208 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124009.log 2024-09-18T01:18:59.208 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125519.log 2024-09-18T01:18:59.209 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32112.log 2024-09-18T01:18:59.209 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.118927.log 2024-09-18T01:18:59.209 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88705.log 2024-09-18T01:18:59.209 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61057.log 2024-09-18T01:18:59.209 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108642.log 2024-09-18T01:18:59.209 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34614.log 2024-09-18T01:18:59.209 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36673.log 2024-09-18T01:18:59.209 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119014.log 2024-09-18T01:18:59.209 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46302.log 2024-09-18T01:18:59.210 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88085.log 2024-09-18T01:18:59.211 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69075.log 2024-09-18T01:18:59.211 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.94673.log 2024-09-18T01:18:59.211 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137873.log 2024-09-18T01:18:59.211 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35200.log 2024-09-18T01:18:59.211 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136360.log 2024-09-18T01:18:59.211 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44724.log 2024-09-18T01:18:59.211 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102406.log 2024-09-18T01:18:59.211 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122014.log 2024-09-18T01:18:59.213 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52158.log 2024-09-18T01:18:59.213 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114073.log 2024-09-18T01:18:59.213 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40444.log 2024-09-18T01:18:59.213 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33419.log 2024-09-18T01:18:59.213 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136820.log 2024-09-18T01:18:59.213 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34424.log 2024-09-18T01:18:59.214 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140824.log 2024-09-18T01:18:59.214 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53627.log 2024-09-18T01:18:59.214 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121679.log 2024-09-18T01:18:59.214 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117695.log 2024-09-18T01:18:59.215 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63948.log 2024-09-18T01:18:59.215 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120596.log 2024-09-18T01:18:59.215 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51011.log 2024-09-18T01:18:59.215 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129741.log 2024-09-18T01:18:59.215 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131531.log 2024-09-18T01:18:59.215 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32422.log 2024-09-18T01:18:59.215 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55606.log 2024-09-18T01:18:59.216 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36738.log 2024-09-18T01:18:59.216 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60328.log 2024-09-18T01:18:59.216 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28995.log 2024-09-18T01:18:59.217 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80658.log 2024-09-18T01:18:59.217 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127784.log 2024-09-18T01:18:59.217 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47462.log 2024-09-18T01:18:59.218 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43740.log 2024-09-18T01:18:59.218 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74857.log 2024-09-18T01:18:59.218 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64196.log 2024-09-18T01:18:59.218 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57267.log 2024-09-18T01:18:59.218 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96168.log 2024-09-18T01:18:59.219 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113491.log 2024-09-18T01:18:59.219 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95393.log 2024-09-18T01:18:59.219 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113492.log 2024-09-18T01:18:59.219 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141933.log 2024-09-18T01:18:59.220 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30505.log 2024-09-18T01:18:59.220 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96768.log 2024-09-18T01:18:59.220 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89592.log 2024-09-18T01:18:59.220 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71265.log 2024-09-18T01:18:59.220 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92813.log 2024-09-18T01:18:59.220 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29780.log 2024-09-18T01:18:59.221 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.104984.log 2024-09-18T01:18:59.221 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133195.log 2024-09-18T01:18:59.221 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89716.log 2024-09-18T01:18:59.221 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104617.log 2024-09-18T01:18:59.222 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46997.log 2024-09-18T01:18:59.222 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132340.log 2024-09-18T01:18:59.222 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107319.log 2024-09-18T01:18:59.222 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118541.log 2024-09-18T01:18:59.223 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61988.log 2024-09-18T01:18:59.223 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78463.log 2024-09-18T01:18:59.223 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138561.log 2024-09-18T01:18:59.224 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45799.log 2024-09-18T01:18:59.224 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87314.log 2024-09-18T01:18:59.224 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46894.log 2024-09-18T01:18:59.224 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90079.log 2024-09-18T01:18:59.225 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56764.log 2024-09-18T01:18:59.225 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47528.log 2024-09-18T01:18:59.225 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125310.log 2024-09-18T01:18:59.226 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104007.log 2024-09-18T01:18:59.226 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45210.log 2024-09-18T01:18:59.226 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.116701.log 2024-09-18T01:18:59.226 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130197.log 2024-09-18T01:18:59.226 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129261.log 2024-09-18T01:18:59.227 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30941.log 2024-09-18T01:18:59.227 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66969.log 2024-09-18T01:18:59.227 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90263.log 2024-09-18T01:18:59.227 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101855.log 2024-09-18T01:18:59.227 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124426.log 2024-09-18T01:18:59.228 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40417.log 2024-09-18T01:18:59.228 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95944.log 2024-09-18T01:18:59.228 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96023.log 2024-09-18T01:18:59.229 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102319.log 2024-09-18T01:18:59.229 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98774.log 2024-09-18T01:18:59.229 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80908.log 2024-09-18T01:18:59.229 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88180.log 2024-09-18T01:18:59.230 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50059.log 2024-09-18T01:18:59.230 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75149.log 2024-09-18T01:18:59.230 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117973.log 2024-09-18T01:18:59.231 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127094.log 2024-09-18T01:18:59.231 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62322.log 2024-09-18T01:18:59.231 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91696.log 2024-09-18T01:18:59.231 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63792.log 2024-09-18T01:18:59.243 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57242.log 2024-09-18T01:18:59.243 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82735.log 2024-09-18T01:18:59.243 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119295.log 2024-09-18T01:18:59.243 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122662.log: /var/log/ceph/cluster1-client.mirror.33795.log: /var/log/ceph/cluster1-client.admin.82735.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129651.log 2024-09-18T01:18:59.243 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110698.log: /var/log/ceph/cluster1-client.mirror.79051.log: /var/log/ceph/cluster1-client.mirror.98837.log: /var/log/ceph/cluster2-client.mirror.119295.log: /var/log/ceph/cluster1-client.mirror.137581.log: gzip/var/log/ceph/cluster2-client.mirror.63085.log: -5 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.127460.log.gz 2024-09-18T01:18:59.243 INFO:teuthology.orchestra.run.smithi049.stderr: --verbose/var/log/ceph/cluster1-client.mirror.100724.log: -- /var/log/ceph/cluster1-client.mirror.115599.log 2024-09-18T01:18:59.243 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.96166.log: /var/log/ceph/cluster2-client.admin.129651.log: /var/log/ceph/cluster1-client.mirror.93024.log: /var/log/ceph/cluster1-client.mirror.134037.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96166.log.gz 2024-09-18T01:18:59.244 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129651.log.gz 2024-09-18T01:18:59.244 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44327.log: /var/log/ceph/cluster2-client.admin.106957.log: /var/log/ceph/cluster2-client.admin.105986.log: /var/log/ceph/cluster1-client.mirror.69494.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35580.log 2024-09-18T01:18:59.244 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82444.log.gz 2024-09-18T01:18:59.244 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.49941.log.gz 2024-09-18T01:18:59.244 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78884.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.94993.log.gz 2024-09-18T01:18:59.244 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112272.log: /var/log/ceph/cluster1-client.mirror.126947.log: /var/log/ceph/cluster1-client.mirror.133575.log: /var/log/ceph/cluster1-client.mirror.116658.log: /var/log/ceph/cluster2-client.admin.62204.log: /var/log/ceph/cluster1-client.mirror.115599.log: /var/log/ceph/cluster1-client.mirror.55141.log: /var/log/ceph/cluster1-client.admin.52484.log: /var/log/ceph/cluster2-client.admin.71936.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62204.log.gz 2024-09-18T01:18:59.244 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99037.log 2024-09-18T01:18:59.244 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101438.log: /var/log/ceph/cluster1-client.mirror.86479.log: /var/log/ceph/cluster1-client.mirror.121004.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.52484.log.gz 2024-09-18T01:18:59.244 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.62723.log: /var/log/ceph/cluster1-client.mirror.81580.log: /var/log/ceph/cluster1-client.mirror.120462.log: /var/log/ceph/cluster1-client.mirror.101773.log: /var/log/ceph/cluster1-client.mirror.35580.log: /var/log/ceph/cluster1-client.mirror.67659.log: /var/log/ceph/cluster1-client.mirror.97893.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.93729.log.gz 2024-09-18T01:18:59.244 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.50587.log: /var/log/ceph/cluster1-client.mirror.89085.log: /var/log/ceph/cluster2-client.admin.117732.log: /var/log/ceph/cluster2-client.admin.142416.log: 67.8% -- replaced with /var/log/ceph/cluster1-client.mirror.86028.log.gz 2024-09-18T01:18:59.244 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56761.log: /var/log/ceph/cluster1-client.mirror.79346.log: gzip -5 --verbose -- 0.0% /var/log/ceph/cluster1-client.mirror.125941.log 2024-09-18T01:18:59.244 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.142416.log.gz/var/log/ceph/cluster1-client.mirror.111640.log: 2024-09-18T01:18:59.245 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.64546.log: /var/log/ceph/cluster1-client.mirror.63878.log: /var/log/ceph/cluster1-client.mirror.70927.log: /var/log/ceph/cluster1-client.mirror.54182.log: /var/log/ceph/cluster1-client.mirror.81204.log: /var/log/ceph/cluster2-client.admin.29754.log: /var/log/ceph/cluster1-client.mirror.59077.log: /var/log/ceph/cluster1-client.mirror.99037.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29754.log.gz 2024-09-18T01:18:59.245 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137033.log: /var/log/ceph/cluster1-client.mirror.119873.log: /var/log/ceph/cluster1-client.admin.30676.log: /var/log/ceph/cluster1-client.mirror.77707.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.105855.log.gz 2024-09-18T01:18:59.245 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106687.log 2024-09-18T01:18:59.245 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.29096.log: /var/log/ceph/cluster1-client.admin.127556.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.127556.log.gz 2024-09-18T01:18:59.245 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.125941.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29096.log.gz 2024-09-18T01:18:59.245 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66443.log: /var/log/ceph/cluster2-client.admin.28742.log: /var/log/ceph/cluster1-client.mirror.70127.log: /var/log/ceph/cluster1-client.mirror.41195.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.83257.log.gz 2024-09-18T01:18:59.245 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.85277.log: /var/log/ceph/cluster1-client.mirror.121728.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28742.log.gz 2024-09-18T01:18:59.245 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.140304.log 2024-09-18T01:18:59.245 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.83654.log.gz/var/log/ceph/cluster1-client.mirror.59791.log: 2024-09-18T01:18:59.245 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.107023.log.gz 2024-09-18T01:18:59.245 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.120286.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.132017.log 2024-09-18T01:18:59.245 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63576.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72495.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88875.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117623.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46263.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56552.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80029.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49723.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96567.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61272.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35315.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30398.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134714.log 2024-09-18T01:18:59.246 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61679.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44054.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95678.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65971.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91764.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90851.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52401.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133450.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65301.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141839.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133788.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98177.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48408.log 2024-09-18T01:18:59.247 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33060.log 2024-09-18T01:18:59.248 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98010.log 2024-09-18T01:18:59.248 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141106.log 2024-09-18T01:18:59.248 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61101.log 2024-09-18T01:18:59.302 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69788.log 2024-09-18T01:18:59.302 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134306.log 2024-09-18T01:18:59.302 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35518.log 2024-09-18T01:18:59.302 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128832.log 2024-09-18T01:18:59.302 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103292.log 2024-09-18T01:18:59.302 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62288.log 2024-09-18T01:18:59.302 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102575.log 2024-09-18T01:18:59.302 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119905.log 2024-09-18T01:18:59.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29196.log 2024-09-18T01:18:59.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115433.log 2024-09-18T01:18:59.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91908.log 2024-09-18T01:18:59.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132874.log 2024-09-18T01:18:59.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77204.log 2024-09-18T01:18:59.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64728.log 2024-09-18T01:18:59.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89926.log 2024-09-18T01:18:59.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138658.log 2024-09-18T01:18:59.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75309.log 2024-09-18T01:18:59.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47840.log 2024-09-18T01:18:59.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67138.log 2024-09-18T01:18:59.303 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96406.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60833.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112237.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136320.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93023.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74190.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120789.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50004.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84451.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105906.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70082.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105306.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134308.log 2024-09-18T01:18:59.304 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80867.log 2024-09-18T01:18:59.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48859.log 2024-09-18T01:18:59.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84981.log 2024-09-18T01:18:59.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47272.log 2024-09-18T01:18:59.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132842.log 2024-09-18T01:18:59.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.94764.log 2024-09-18T01:18:59.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91614.log 2024-09-18T01:18:59.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48158.log 2024-09-18T01:18:59.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115639.log 2024-09-18T01:18:59.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81539.log 2024-09-18T01:18:59.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68018.log 2024-09-18T01:18:59.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82253.log 2024-09-18T01:18:59.305 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86238.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76460.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99498.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30819.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51019.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114000.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92242.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77916.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130124.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55224.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74994.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56217.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129281.log 2024-09-18T01:18:59.306 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114128.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77789.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.140878.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117696.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96499.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66234.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.139893.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94458.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73909.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112440.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124845.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77498.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60974.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100638.log 2024-09-18T01:18:59.307 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33375.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71805.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37973.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135941.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66967.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90053.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76729.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112273.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103333.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74552.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30976.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112168.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.107592.log 2024-09-18T01:18:59.308 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140838.log 2024-09-18T01:18:59.309 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.140303.log 2024-09-18T01:18:59.309 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108754.log 2024-09-18T01:18:59.309 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63106.log 2024-09-18T01:18:59.309 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31180.log 2024-09-18T01:18:59.309 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67261.log 2024-09-18T01:18:59.309 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134710.log 2024-09-18T01:18:59.309 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141993.log 2024-09-18T01:18:59.309 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103629.log 2024-09-18T01:18:59.309 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54908.log 2024-09-18T01:18:59.309 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39274.log 2024-09-18T01:18:59.309 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140606.log 2024-09-18T01:18:59.309 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124259.log 2024-09-18T01:18:59.310 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40987.log 2024-09-18T01:18:59.310 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40039.log 2024-09-18T01:18:59.310 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44832.log 2024-09-18T01:18:59.310 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137200.log 2024-09-18T01:18:59.310 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130218.log 2024-09-18T01:18:59.310 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28665.log 2024-09-18T01:18:59.310 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122722.log 2024-09-18T01:18:59.310 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46472.log 2024-09-18T01:18:59.310 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119584.log 2024-09-18T01:18:59.310 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58616.log 2024-09-18T01:18:59.310 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98105.log 2024-09-18T01:18:59.310 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131913.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40926.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73310.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41154.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77370.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50998.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30204.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56218.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67516.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93992.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32640.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37157.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52633.log 2024-09-18T01:18:59.311 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66634.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100304.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125730.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137832.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88748.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41876.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128645.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135981.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81918.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138666.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81665.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133406.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98527.log 2024-09-18T01:18:59.312 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61468.log 2024-09-18T01:18:59.313 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64350.log 2024-09-18T01:18:59.313 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33794.log 2024-09-18T01:18:59.313 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98798.log 2024-09-18T01:18:59.313 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114338.log 2024-09-18T01:18:59.313 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57394.log 2024-09-18T01:18:59.313 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140681.log 2024-09-18T01:18:59.313 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99925.log 2024-09-18T01:18:59.313 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48261.log 2024-09-18T01:18:59.313 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69999.log 2024-09-18T01:18:59.313 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74210.log 2024-09-18T01:18:59.313 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135130.log 2024-09-18T01:18:59.313 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44200.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.121360.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40303.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42048.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91841.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73868.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34319.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78755.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100385.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60762.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46850.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85478.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57602.log 2024-09-18T01:18:59.314 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68652.log 2024-09-18T01:18:59.315 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96909.log 2024-09-18T01:18:59.315 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114590.log 2024-09-18T01:18:59.315 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.140215.log 2024-09-18T01:18:59.315 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55014.log 2024-09-18T01:18:59.315 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57772.log 2024-09-18T01:18:59.315 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71806.log 2024-09-18T01:18:59.315 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78883.log 2024-09-18T01:18:59.315 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129354.log 2024-09-18T01:18:59.324 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70629.log 2024-09-18T01:18:59.324 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138004.log 2024-09-18T01:18:59.324 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98842.log 2024-09-18T01:18:59.324 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60096.log 2024-09-18T01:18:59.324 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127116.log 2024-09-18T01:18:59.324 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.107934.log 2024-09-18T01:18:59.324 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75524.log 2024-09-18T01:18:59.324 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31355.log 2024-09-18T01:18:59.324 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125520.log 2024-09-18T01:18:59.324 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56467.log 2024-09-18T01:18:59.324 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97137.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98484.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54807.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58154.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50111.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78839.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72128.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125186.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98049.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140812.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108545.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134141.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85572.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39689.log 2024-09-18T01:18:59.325 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57945.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31179.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113744.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124761.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140828.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113788.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123629.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55434.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36253.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77499.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39962.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31550.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34737.log 2024-09-18T01:18:59.326 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40356.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33627.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33313.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123000.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63665.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100764.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67390.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102574.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.132019.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45774.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93509.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114337.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82126.log 2024-09-18T01:18:59.327 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58030.log 2024-09-18T01:18:59.328 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98154.log 2024-09-18T01:18:59.328 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111851.log 2024-09-18T01:18:59.328 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55328.log 2024-09-18T01:18:59.328 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50980.log 2024-09-18T01:18:59.328 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50320.log 2024-09-18T01:18:59.328 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35727.log 2024-09-18T01:18:59.328 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47062.log 2024-09-18T01:18:59.328 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103838.log 2024-09-18T01:18:59.328 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92393.log 2024-09-18T01:18:59.328 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55982.log 2024-09-18T01:18:59.328 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45000.log 2024-09-18T01:18:59.328 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45251.log 2024-09-18T01:18:59.329 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115640.log 2024-09-18T01:18:59.329 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108343.log 2024-09-18T01:18:59.329 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41704.log 2024-09-18T01:18:59.329 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56194.log 2024-09-18T01:18:59.329 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103462.log 2024-09-18T01:18:59.329 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117486.log 2024-09-18T01:18:59.329 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103461.log 2024-09-18T01:18:59.329 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32316.log 2024-09-18T01:18:59.329 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43574.log 2024-09-18T01:18:59.329 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138401.log 2024-09-18T01:18:59.329 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131621.log 2024-09-18T01:18:59.329 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35475.log 2024-09-18T01:18:59.330 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86776.log 2024-09-18T01:18:59.330 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88915.log 2024-09-18T01:18:59.330 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69766.log 2024-09-18T01:18:59.330 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63771.log 2024-09-18T01:18:59.330 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42113.log 2024-09-18T01:18:59.330 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70929.log 2024-09-18T01:18:59.333 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36949.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78000.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70885.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121845.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61041.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137664.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118880.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29925.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73126.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115758.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139574.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140806.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112588.log 2024-09-18T01:18:59.334 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100682.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41274.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100553.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87598.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133667.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43632.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56007.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60712.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100806.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59983.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58532.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45638.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123165.log 2024-09-18T01:18:59.335 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79301.log 2024-09-18T01:18:59.336 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100346.log 2024-09-18T01:18:59.336 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96156.log 2024-09-18T01:18:59.336 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136191.log 2024-09-18T01:18:59.336 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111162.log 2024-09-18T01:18:59.336 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111390.log 2024-09-18T01:18:59.336 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134500.log 2024-09-18T01:18:59.336 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138082.log 2024-09-18T01:18:59.336 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84884.log 2024-09-18T01:18:59.336 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29652.log 2024-09-18T01:18:59.339 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48470.log 2024-09-18T01:18:59.339 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134417.log 2024-09-18T01:18:59.339 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50128.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49587.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55118.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51300.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43674.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72808.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138621.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46196.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132264.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43741.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36213.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34822.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63407.log 2024-09-18T01:18:59.340 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86084.log 2024-09-18T01:18:59.341 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54250.log 2024-09-18T01:18:59.341 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114421.log 2024-09-18T01:18:59.341 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130674.log 2024-09-18T01:18:59.341 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93379.log 2024-09-18T01:18:59.341 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43129.log 2024-09-18T01:18:59.341 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78335.log 2024-09-18T01:18:59.343 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131883.log 2024-09-18T01:18:59.343 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126085.log 2024-09-18T01:18:59.343 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36715.log 2024-09-18T01:18:59.343 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102619.log 2024-09-18T01:18:59.343 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136613.log 2024-09-18T01:18:59.344 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71055.log 2024-09-18T01:18:59.344 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96593.log 2024-09-18T01:18:59.344 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136321.log 2024-09-18T01:18:59.344 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72078.log 2024-09-18T01:18:59.344 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100134.log 2024-09-18T01:18:59.344 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92454.log 2024-09-18T01:18:59.344 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46365.log 2024-09-18T01:18:59.344 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66377.log 2024-09-18T01:18:59.344 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74251.log 2024-09-18T01:18:59.346 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62583.log 2024-09-18T01:18:59.347 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.121134.log 2024-09-18T01:18:59.347 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78464.log 2024-09-18T01:18:59.347 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123564.log 2024-09-18T01:18:59.347 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34005.log 2024-09-18T01:18:59.347 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36359.log 2024-09-18T01:18:59.347 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130916.log 2024-09-18T01:18:59.347 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81581.log 2024-09-18T01:18:59.347 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69412.log 2024-09-18T01:18:59.347 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95570.log 2024-09-18T01:18:59.347 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112948.log 2024-09-18T01:18:59.347 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39706.log 2024-09-18T01:18:59.349 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.30015.log 2024-09-18T01:18:59.349 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65885.log 2024-09-18T01:18:59.350 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120661.log 2024-09-18T01:18:59.350 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136951.log 2024-09-18T01:18:59.350 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83543.log 2024-09-18T01:18:59.350 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97845.log 2024-09-18T01:18:59.350 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30387.log 2024-09-18T01:18:59.350 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41174.log 2024-09-18T01:18:59.350 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83661.log 2024-09-18T01:18:59.350 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89212.log 2024-09-18T01:18:59.353 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127242.log 2024-09-18T01:18:59.353 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29146.log 2024-09-18T01:18:59.353 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134078.log 2024-09-18T01:18:59.353 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123840.log 2024-09-18T01:18:59.353 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128322.log 2024-09-18T01:18:59.353 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61550.log 2024-09-18T01:18:59.354 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31601.log 2024-09-18T01:18:59.354 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59349.log 2024-09-18T01:18:59.354 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73449.log 2024-09-18T01:18:59.354 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75698.log 2024-09-18T01:18:59.354 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108955.log 2024-09-18T01:18:59.354 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95563.log 2024-09-18T01:18:59.354 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57184.log 2024-09-18T01:18:59.354 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110586.log 2024-09-18T01:18:59.355 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113787.log 2024-09-18T01:18:59.355 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89927.log 2024-09-18T01:18:59.355 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106730.log 2024-09-18T01:18:59.356 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132725.log 2024-09-18T01:18:59.356 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43781.log 2024-09-18T01:18:59.356 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50490.log 2024-09-18T01:18:59.356 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61542.log 2024-09-18T01:18:59.356 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95237.log 2024-09-18T01:18:59.358 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112859.log 2024-09-18T01:18:59.358 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139759.log 2024-09-18T01:18:59.358 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90705.log 2024-09-18T01:18:59.358 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32893.log 2024-09-18T01:18:59.358 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75659.log 2024-09-18T01:18:59.358 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131085.log 2024-09-18T01:18:59.358 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70816.log 2024-09-18T01:18:59.358 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70294.log 2024-09-18T01:18:59.358 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35410.log 2024-09-18T01:18:59.359 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130633.log 2024-09-18T01:18:59.359 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122803.log 2024-09-18T01:18:59.360 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112523.log 2024-09-18T01:18:59.360 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50642.log 2024-09-18T01:18:59.360 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52495.log 2024-09-18T01:18:59.360 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44538.log 2024-09-18T01:18:59.361 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67767.log 2024-09-18T01:18:59.361 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43908.log 2024-09-18T01:18:59.361 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122958.log 2024-09-18T01:18:59.361 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106756.log 2024-09-18T01:18:59.361 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119757.log 2024-09-18T01:18:59.363 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58954.log 2024-09-18T01:18:59.363 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138887.log 2024-09-18T01:18:59.363 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.116869.log 2024-09-18T01:18:59.363 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75240.log 2024-09-18T01:18:59.363 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78757.log 2024-09-18T01:18:59.363 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36505.log 2024-09-18T01:18:59.363 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67597.log 2024-09-18T01:18:59.363 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44160.log 2024-09-18T01:18:59.365 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30618.log 2024-09-18T01:18:59.365 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38928.log 2024-09-18T01:18:59.365 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87562.log 2024-09-18T01:18:59.365 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55270.log 2024-09-18T01:18:59.365 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67262.log 2024-09-18T01:18:59.365 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46220.log 2024-09-18T01:18:59.365 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126925.log 2024-09-18T01:18:59.366 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60225.log 2024-09-18T01:18:59.366 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131596.log 2024-09-18T01:18:59.366 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119755.log 2024-09-18T01:18:59.366 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122955.log 2024-09-18T01:18:59.366 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99091.log 2024-09-18T01:18:59.367 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88179.log 2024-09-18T01:18:59.367 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51797.log 2024-09-18T01:18:59.367 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130653.log 2024-09-18T01:18:59.367 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109470.log 2024-09-18T01:18:59.368 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109316.log 2024-09-18T01:18:59.368 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133787.log 2024-09-18T01:18:59.368 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110375.log 2024-09-18T01:18:59.368 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87093.log 2024-09-18T01:18:59.370 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57348.log 2024-09-18T01:18:59.370 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95648.log 2024-09-18T01:18:59.370 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127617.log 2024-09-18T01:18:59.370 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105735.log 2024-09-18T01:18:59.370 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31857.log 2024-09-18T01:18:59.370 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111539.log 2024-09-18T01:18:59.371 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84561.log 2024-09-18T01:18:59.371 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100345.log 2024-09-18T01:18:59.371 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124215.log 2024-09-18T01:18:59.371 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91195.log 2024-09-18T01:18:59.372 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48471.log 2024-09-18T01:18:59.372 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118391.log 2024-09-18T01:18:59.373 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124637.log 2024-09-18T01:18:59.373 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92370.log 2024-09-18T01:18:59.373 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52601.log 2024-09-18T01:18:59.373 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32849.log 2024-09-18T01:18:59.373 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55879.log 2024-09-18T01:18:59.374 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73037.log 2024-09-18T01:18:59.374 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128712.log 2024-09-18T01:18:59.374 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109551.log 2024-09-18T01:18:59.375 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131047.log 2024-09-18T01:18:59.375 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71840.log 2024-09-18T01:18:59.375 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125731.log 2024-09-18T01:18:59.375 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68291.log 2024-09-18T01:18:59.375 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48912.log 2024-09-18T01:18:59.376 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127994.log 2024-09-18T01:18:59.376 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108913.log 2024-09-18T01:18:59.376 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75760.log 2024-09-18T01:18:59.376 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41385.log 2024-09-18T01:18:59.377 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88246.log 2024-09-18T01:18:59.377 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63706.log 2024-09-18T01:18:59.377 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.118558.log 2024-09-18T01:18:59.378 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131556.log 2024-09-18T01:18:59.378 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78673.log 2024-09-18T01:18:59.378 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119128.log 2024-09-18T01:18:59.379 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70754.log 2024-09-18T01:18:59.379 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117694.log 2024-09-18T01:18:59.379 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96727.log 2024-09-18T01:18:59.379 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60834.log 2024-09-18T01:18:59.380 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118652.log 2024-09-18T01:18:59.380 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39737.log 2024-09-18T01:18:59.380 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60104.log 2024-09-18T01:18:59.381 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59559.log 2024-09-18T01:18:59.381 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103544.log 2024-09-18T01:18:59.381 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114064.log 2024-09-18T01:18:59.382 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123354.log 2024-09-18T01:18:59.382 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65046.log 2024-09-18T01:18:59.382 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82731.log 2024-09-18T01:18:59.382 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35728.log 2024-09-18T01:18:59.383 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53175.log 2024-09-18T01:18:59.383 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35685.log 2024-09-18T01:18:59.383 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93000.log 2024-09-18T01:18:59.383 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37963.log 2024-09-18T01:18:59.384 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52463.log 2024-09-18T01:18:59.384 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95685.log 2024-09-18T01:18:59.384 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103754.log 2024-09-18T01:18:59.385 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106517.log 2024-09-18T01:18:59.385 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110141.log 2024-09-18T01:18:59.385 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68185.log 2024-09-18T01:18:59.386 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132779.log 2024-09-18T01:18:59.386 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90346.log 2024-09-18T01:18:59.386 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46684.log 2024-09-18T01:18:59.386 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134841.log 2024-09-18T01:18:59.387 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139539.log 2024-09-18T01:18:59.387 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117138.log 2024-09-18T01:18:59.388 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47571.log 2024-09-18T01:18:59.388 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114801.log 2024-09-18T01:18:59.388 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109111.log 2024-09-18T01:18:59.389 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36842.log 2024-09-18T01:18:59.390 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48053.log 2024-09-18T01:18:59.390 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119657.log 2024-09-18T01:18:59.390 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93083.log 2024-09-18T01:18:59.390 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47780.log 2024-09-18T01:18:59.390 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30715.log 2024-09-18T01:18:59.391 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141535.log 2024-09-18T01:18:59.391 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69789.log 2024-09-18T01:18:59.391 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85873.log 2024-09-18T01:18:59.391 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54674.log 2024-09-18T01:18:59.391 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46724.log 2024-09-18T01:18:59.392 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29959.log 2024-09-18T01:18:59.392 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124342.log 2024-09-18T01:18:59.392 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37995.log 2024-09-18T01:18:59.393 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119543.log 2024-09-18T01:18:59.393 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119087.log 2024-09-18T01:18:59.393 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55269.log 2024-09-18T01:18:59.394 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120525.log 2024-09-18T01:18:59.394 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77047.log 2024-09-18T01:18:59.395 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64934.log 2024-09-18T01:18:59.395 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66274.log 2024-09-18T01:18:59.395 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65625.log 2024-09-18T01:18:59.395 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123588.log 2024-09-18T01:18:59.395 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34676.log 2024-09-18T01:18:59.396 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123795.log 2024-09-18T01:18:59.396 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41741.log 2024-09-18T01:18:59.396 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76469.log 2024-09-18T01:18:59.397 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109003.log 2024-09-18T01:18:59.397 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48118.log 2024-09-18T01:18:59.397 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.29986.log 2024-09-18T01:18:59.397 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117021.log 2024-09-18T01:18:59.398 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97894.log 2024-09-18T01:18:59.398 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140810.log 2024-09-18T01:18:59.398 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77497.log 2024-09-18T01:18:59.399 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106378.log 2024-09-18T01:18:59.399 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47421.log 2024-09-18T01:18:59.399 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86107.log 2024-09-18T01:18:59.399 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69160.log 2024-09-18T01:18:59.400 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58994.log 2024-09-18T01:18:59.400 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70712.log 2024-09-18T01:18:59.401 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99119.log 2024-09-18T01:18:59.401 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58299.log 2024-09-18T01:18:59.401 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.140965.log 2024-09-18T01:18:59.401 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71989.log 2024-09-18T01:18:59.401 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98085.log 2024-09-18T01:18:59.402 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32850.log 2024-09-18T01:18:59.402 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104092.log 2024-09-18T01:18:59.403 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98536.log 2024-09-18T01:18:59.403 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68524.log 2024-09-18T01:18:59.403 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44537.log 2024-09-18T01:18:59.403 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126527.log 2024-09-18T01:18:59.404 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64299.log 2024-09-18T01:18:59.404 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41060.log 2024-09-18T01:18:59.404 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129472.log 2024-09-18T01:18:59.405 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40168.log 2024-09-18T01:18:59.405 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94945.log 2024-09-18T01:18:59.405 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111327.log 2024-09-18T01:18:59.405 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138497.log 2024-09-18T01:18:59.407 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99159.log 2024-09-18T01:18:59.407 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140945.log 2024-09-18T01:18:59.407 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114700.log 2024-09-18T01:18:59.407 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78167.log 2024-09-18T01:18:59.407 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46306.log 2024-09-18T01:18:59.408 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86582.log 2024-09-18T01:18:59.408 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56088.log 2024-09-18T01:18:59.408 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109675.log 2024-09-18T01:18:59.408 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67935.log 2024-09-18T01:18:59.409 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61656.log 2024-09-18T01:18:59.410 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138318.log 2024-09-18T01:18:59.410 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67601.log 2024-09-18T01:18:59.411 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44895.log 2024-09-18T01:18:59.411 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43884.log 2024-09-18T01:18:59.411 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29071.log 2024-09-18T01:18:59.413 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31603.log 2024-09-18T01:18:59.413 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128314.log 2024-09-18T01:18:59.413 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54030.log 2024-09-18T01:18:59.414 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40625.log 2024-09-18T01:18:59.414 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133237.log 2024-09-18T01:18:59.414 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84760.log 2024-09-18T01:18:59.414 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91485.log 2024-09-18T01:18:59.414 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92873.log 2024-09-18T01:18:59.414 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64352.log 2024-09-18T01:18:59.414 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89656.log 2024-09-18T01:18:59.414 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142065.log 2024-09-18T01:18:59.414 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128926.log 2024-09-18T01:18:59.415 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123210.log 2024-09-18T01:18:59.415 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82602.log 2024-09-18T01:18:59.415 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58533.log 2024-09-18T01:18:59.415 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99539.log 2024-09-18T01:18:59.415 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92456.log 2024-09-18T01:18:59.416 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45883.log 2024-09-18T01:18:59.416 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124343.log 2024-09-18T01:18:59.416 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119409.log 2024-09-18T01:18:59.416 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39752.log 2024-09-18T01:18:59.416 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125265.log 2024-09-18T01:18:59.417 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63504.log 2024-09-18T01:18:59.417 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.138830.log 2024-09-18T01:18:59.418 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73552.log 2024-09-18T01:18:59.418 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120115.log 2024-09-18T01:18:59.418 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136740.log 2024-09-18T01:18:59.418 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106109.log 2024-09-18T01:18:59.419 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123501.log 2024-09-18T01:18:59.419 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101645.log 2024-09-18T01:18:59.419 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80448.log 2024-09-18T01:18:59.419 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.129162.log 2024-09-18T01:18:59.420 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129643.log 2024-09-18T01:18:59.420 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104553.log 2024-09-18T01:18:59.421 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94485.log 2024-09-18T01:18:59.421 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32892.log 2024-09-18T01:18:59.421 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125606.log 2024-09-18T01:18:59.421 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41933.log 2024-09-18T01:18:59.422 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96339.log 2024-09-18T01:18:59.422 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67809.log 2024-09-18T01:18:59.422 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105137.log 2024-09-18T01:18:59.423 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75997.log 2024-09-18T01:18:59.423 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87876.log 2024-09-18T01:18:59.423 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108957.log 2024-09-18T01:18:59.423 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86116.log 2024-09-18T01:18:59.425 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48260.log 2024-09-18T01:18:59.425 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74479.log 2024-09-18T01:18:59.425 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117134.log 2024-09-18T01:18:59.425 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52538.log 2024-09-18T01:18:59.425 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119714.log 2024-09-18T01:18:59.426 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73038.log 2024-09-18T01:18:59.426 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.98573.log 2024-09-18T01:18:59.426 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.94080.log 2024-09-18T01:18:59.426 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51414.log 2024-09-18T01:18:59.427 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78209.log 2024-09-18T01:18:59.427 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131143.log 2024-09-18T01:18:59.427 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33899.log 2024-09-18T01:18:59.427 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138127.log 2024-09-18T01:18:59.427 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54849.log 2024-09-18T01:18:59.428 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38397.log 2024-09-18T01:18:59.428 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122324.log 2024-09-18T01:18:59.429 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30167.log 2024-09-18T01:18:59.429 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140842.log 2024-09-18T01:18:59.429 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87831.log 2024-09-18T01:18:59.429 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83607.log 2024-09-18T01:18:59.430 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135050.log 2024-09-18T01:18:59.430 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71694.log 2024-09-18T01:18:59.430 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67556.log 2024-09-18T01:18:59.431 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88286.log 2024-09-18T01:18:59.431 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93815.log 2024-09-18T01:18:59.431 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116206.log 2024-09-18T01:18:59.432 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132170.log 2024-09-18T01:18:59.432 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92536.log 2024-09-18T01:18:59.432 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30777.log 2024-09-18T01:18:59.433 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136531.log 2024-09-18T01:18:59.433 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50420.log 2024-09-18T01:18:59.433 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62043.log 2024-09-18T01:18:59.433 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59625.log 2024-09-18T01:18:59.434 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58660.log 2024-09-18T01:18:59.434 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37834.log 2024-09-18T01:18:59.435 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63907.log 2024-09-18T01:18:59.435 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70907.log 2024-09-18T01:18:59.435 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108436.log 2024-09-18T01:18:59.435 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48515.log 2024-09-18T01:18:59.436 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85759.log 2024-09-18T01:18:59.436 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.94766.log 2024-09-18T01:18:59.437 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35474.log 2024-09-18T01:18:59.437 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32465.log 2024-09-18T01:18:59.438 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84869.log 2024-09-18T01:18:59.438 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34320.log 2024-09-18T01:18:59.438 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120418.log 2024-09-18T01:18:59.439 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61022.log 2024-09-18T01:18:59.439 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49101.log 2024-09-18T01:18:59.439 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136150.log 2024-09-18T01:18:59.439 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106724.log 2024-09-18T01:18:59.439 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30932.log 2024-09-18T01:18:59.440 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78294.log 2024-09-18T01:18:59.440 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65947.log 2024-09-18T01:18:59.441 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35054.log 2024-09-18T01:18:59.441 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87726.log 2024-09-18T01:18:59.441 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139756.log 2024-09-18T01:18:59.441 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139137.log 2024-09-18T01:18:59.441 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109242.log 2024-09-18T01:18:59.442 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140710.log 2024-09-18T01:18:59.442 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80196.log 2024-09-18T01:18:59.443 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108001.log 2024-09-18T01:18:59.443 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69579.log 2024-09-18T01:18:59.443 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128979.log 2024-09-18T01:18:59.443 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97374.log 2024-09-18T01:18:59.444 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74760.log 2024-09-18T01:18:59.444 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37555.log 2024-09-18T01:18:59.444 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118859.log 2024-09-18T01:18:59.445 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92180.log 2024-09-18T01:18:59.445 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46726.log 2024-09-18T01:18:59.445 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136192.log 2024-09-18T01:18:59.445 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115223.log 2024-09-18T01:18:59.446 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127052.log 2024-09-18T01:18:59.446 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138357.log 2024-09-18T01:18:59.447 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80236.log 2024-09-18T01:18:59.447 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82584.log 2024-09-18T01:18:59.447 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137831.log 2024-09-18T01:18:59.448 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36212.log 2024-09-18T01:18:59.448 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79259.log 2024-09-18T01:18:59.448 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100973.log 2024-09-18T01:18:59.448 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76263.log 2024-09-18T01:18:59.449 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71267.log 2024-09-18T01:18:59.449 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140281.log 2024-09-18T01:18:59.449 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72960.log 2024-09-18T01:18:59.449 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31901.log 2024-09-18T01:18:59.450 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101731.log 2024-09-18T01:18:59.450 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104682.log 2024-09-18T01:18:59.450 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103294.log 2024-09-18T01:18:59.451 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94033.log 2024-09-18T01:18:59.451 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43992.log 2024-09-18T01:18:59.451 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88001.log 2024-09-18T01:18:59.452 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96654.log 2024-09-18T01:18:59.452 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121416.log 2024-09-18T01:18:59.452 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42156.log 2024-09-18T01:18:59.453 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117018.log 2024-09-18T01:18:59.453 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33900.log 2024-09-18T01:18:59.454 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.140040.log 2024-09-18T01:18:59.454 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123167.log 2024-09-18T01:18:59.454 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84262.log 2024-09-18T01:18:59.454 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103207.log 2024-09-18T01:18:59.455 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131190.log 2024-09-18T01:18:59.455 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126317.log 2024-09-18T01:18:59.455 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115979.log 2024-09-18T01:18:59.456 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92956.log 2024-09-18T01:18:59.456 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104849.log 2024-09-18T01:18:59.456 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138191.log 2024-09-18T01:18:59.457 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76017.log 2024-09-18T01:18:59.457 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119637.log 2024-09-18T01:18:59.457 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105789.log 2024-09-18T01:18:59.458 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129469.log 2024-09-18T01:18:59.458 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48664.log 2024-09-18T01:18:59.458 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64804.log 2024-09-18T01:18:59.458 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115854.log 2024-09-18T01:18:59.459 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108513.log 2024-09-18T01:18:59.459 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128102.log 2024-09-18T01:18:59.459 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65082.log 2024-09-18T01:18:59.460 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86414.log 2024-09-18T01:18:59.460 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90769.log 2024-09-18T01:18:59.460 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42086.log 2024-09-18T01:18:59.461 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74137.log 2024-09-18T01:18:59.461 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133113.log 2024-09-18T01:18:59.461 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54616.log 2024-09-18T01:18:59.462 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83009.log 2024-09-18T01:18:59.462 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106949.log 2024-09-18T01:18:59.462 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122323.log 2024-09-18T01:18:59.463 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37263.log 2024-09-18T01:18:59.463 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67869.log 2024-09-18T01:18:59.463 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93442.log 2024-09-18T01:18:59.464 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133997.log 2024-09-18T01:18:59.464 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31095.log 2024-09-18T01:18:59.464 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108887.log 2024-09-18T01:18:59.465 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113856.log 2024-09-18T01:18:59.465 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128060.log 2024-09-18T01:18:59.466 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117589.log 2024-09-18T01:18:59.466 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121415.log 2024-09-18T01:18:59.467 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138767.log 2024-09-18T01:18:59.467 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112439.log 2024-09-18T01:18:59.467 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117972.log 2024-09-18T01:18:59.467 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47631.log 2024-09-18T01:18:59.467 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40786.log 2024-09-18T01:18:59.469 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113221.log 2024-09-18T01:18:59.469 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36360.log 2024-09-18T01:18:59.469 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89168.log 2024-09-18T01:18:59.469 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31970.log 2024-09-18T01:18:59.469 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127456.log 2024-09-18T01:18:59.469 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97740.log 2024-09-18T01:18:59.470 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29806.log 2024-09-18T01:18:59.471 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75838.log 2024-09-18T01:18:59.471 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120953.log 2024-09-18T01:18:59.471 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101353.log 2024-09-18T01:18:59.471 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94721.log 2024-09-18T01:18:59.472 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96043.log 2024-09-18T01:18:59.472 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116282.log 2024-09-18T01:18:59.472 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96385.log 2024-09-18T01:18:59.472 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117268.log 2024-09-18T01:18:59.473 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32579.log 2024-09-18T01:18:59.473 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89422.log 2024-09-18T01:18:59.474 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81706.log 2024-09-18T01:18:59.474 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77664.log 2024-09-18T01:18:59.474 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71254.log 2024-09-18T01:18:59.475 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112482.log 2024-09-18T01:18:59.475 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133070.log 2024-09-18T01:18:59.475 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53327.log 2024-09-18T01:18:59.475 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117377.log 2024-09-18T01:18:59.476 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100848.log 2024-09-18T01:18:59.476 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136193.log 2024-09-18T01:18:59.476 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46938.log 2024-09-18T01:18:59.477 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68864.log 2024-09-18T01:18:59.477 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58721.log 2024-09-18T01:18:59.477 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70396.log 2024-09-18T01:18:59.478 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81877.log 2024-09-18T01:18:59.478 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53435.log 2024-09-18T01:18:59.478 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50463.log 2024-09-18T01:18:59.478 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85075.log 2024-09-18T01:18:59.479 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30225.log 2024-09-18T01:18:59.479 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107638.log 2024-09-18T01:18:59.479 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32998.log 2024-09-18T01:18:59.480 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58112.log 2024-09-18T01:18:59.480 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59857.log 2024-09-18T01:18:59.480 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74345.log 2024-09-18T01:18:59.480 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103166.log 2024-09-18T01:18:59.481 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85645.log 2024-09-18T01:18:59.481 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134630.log 2024-09-18T01:18:59.481 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40101.log 2024-09-18T01:18:59.482 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61387.log 2024-09-18T01:18:59.482 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86727.log 2024-09-18T01:18:59.483 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124194.log 2024-09-18T01:18:59.483 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120584.log 2024-09-18T01:18:59.483 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77289.log 2024-09-18T01:18:59.484 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56343.log 2024-09-18T01:18:59.484 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123386.log 2024-09-18T01:18:59.484 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140904.log 2024-09-18T01:18:59.485 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63612.log 2024-09-18T01:18:59.485 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70462.log 2024-09-18T01:18:59.485 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90283.log 2024-09-18T01:18:59.485 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84878.log 2024-09-18T01:18:59.486 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136615.log 2024-09-18T01:18:59.486 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.121358.log 2024-09-18T01:18:59.486 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63267.log 2024-09-18T01:18:59.487 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67683.log 2024-09-18T01:18:59.487 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116214.log 2024-09-18T01:18:59.487 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66315.log 2024-09-18T01:18:59.487 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108476.log 2024-09-18T01:18:59.488 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121108.log 2024-09-18T01:18:59.489 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44177.log 2024-09-18T01:18:59.489 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65372.log 2024-09-18T01:18:59.489 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45841.log 2024-09-18T01:18:59.490 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110831.log 2024-09-18T01:18:59.490 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129969.log 2024-09-18T01:18:59.490 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128698.log 2024-09-18T01:18:59.490 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94804.log 2024-09-18T01:18:59.490 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136407.log 2024-09-18T01:18:59.491 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53152.log 2024-09-18T01:18:59.492 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95695.log 2024-09-18T01:18:59.492 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47206.log 2024-09-18T01:18:59.493 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110447.log 2024-09-18T01:18:59.493 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35055.log 2024-09-18T01:18:59.493 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41283.log 2024-09-18T01:18:59.494 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126445.log 2024-09-18T01:18:59.494 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106738.log 2024-09-18T01:18:59.494 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.119383.log 2024-09-18T01:18:59.494 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70673.log 2024-09-18T01:18:59.494 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69287.log 2024-09-18T01:18:59.495 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52575.log 2024-09-18T01:18:59.495 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124681.log 2024-09-18T01:18:59.496 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141793.log 2024-09-18T01:18:59.496 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107296.log 2024-09-18T01:18:59.497 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29618.log 2024-09-18T01:18:59.497 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50793.log 2024-09-18T01:18:59.497 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121541.log 2024-09-18T01:18:59.497 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49792.log 2024-09-18T01:18:59.498 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.105752.log 2024-09-18T01:18:59.498 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102277.log 2024-09-18T01:18:59.498 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80237.log 2024-09-18T01:18:59.498 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79435.log 2024-09-18T01:18:59.499 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42308.log 2024-09-18T01:18:59.499 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29986.log 2024-09-18T01:18:59.499 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54402.log 2024-09-18T01:18:59.500 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47038.log 2024-09-18T01:18:59.501 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93971.log 2024-09-18T01:18:59.501 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83944.log 2024-09-18T01:18:59.501 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138251.log 2024-09-18T01:18:59.502 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37473.log 2024-09-18T01:18:59.502 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81496.log 2024-09-18T01:18:59.503 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66097.log 2024-09-18T01:18:59.503 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84865.log 2024-09-18T01:18:59.503 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42654.log 2024-09-18T01:18:59.504 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34572.log 2024-09-18T01:18:59.504 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92243.log 2024-09-18T01:18:59.504 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30227.log 2024-09-18T01:18:59.505 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81119.log 2024-09-18T01:18:59.505 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81749.log 2024-09-18T01:18:59.506 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107172.log 2024-09-18T01:18:59.506 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33772.log 2024-09-18T01:18:59.507 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121500.log 2024-09-18T01:18:59.507 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30665.log 2024-09-18T01:18:59.507 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90557.log 2024-09-18T01:18:59.508 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129647.log 2024-09-18T01:18:59.508 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60200.log 2024-09-18T01:18:59.508 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117784.log 2024-09-18T01:18:59.509 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89296.log 2024-09-18T01:18:59.509 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133912.log 2024-09-18T01:18:59.510 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123712.log 2024-09-18T01:18:59.511 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64634.log 2024-09-18T01:18:59.511 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32359.log 2024-09-18T01:18:59.511 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59921.log 2024-09-18T01:18:59.511 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88396.log 2024-09-18T01:18:59.512 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58575.log 2024-09-18T01:18:59.512 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69074.log 2024-09-18T01:18:59.512 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55480.log 2024-09-18T01:18:59.512 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102149.log 2024-09-18T01:18:59.513 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74050.log 2024-09-18T01:18:59.513 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31488.log 2024-09-18T01:18:59.513 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88813.log 2024-09-18T01:18:59.514 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66400.log 2024-09-18T01:18:59.514 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124614.log 2024-09-18T01:18:59.514 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125393.log 2024-09-18T01:18:59.514 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96795.log 2024-09-18T01:18:59.515 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43633.log 2024-09-18T01:18:59.515 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88181.log 2024-09-18T01:18:59.516 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82402.log 2024-09-18T01:18:59.516 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129585.log 2024-09-18T01:18:59.516 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56047.log 2024-09-18T01:18:59.517 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49046.log 2024-09-18T01:18:59.517 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110951.log 2024-09-18T01:18:59.518 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63527.log 2024-09-18T01:18:59.518 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121867.log 2024-09-18T01:18:59.518 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135554.log 2024-09-18T01:18:59.518 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52437.log 2024-09-18T01:18:59.519 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94992.log 2024-09-18T01:18:59.519 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139751.log 2024-09-18T01:18:59.519 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29635.log 2024-09-18T01:18:59.520 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69833.log 2024-09-18T01:18:59.520 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85505.log 2024-09-18T01:18:59.521 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50699.log 2024-09-18T01:18:59.521 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45040.log 2024-09-18T01:18:59.521 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61039.log 2024-09-18T01:18:59.522 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63840.log 2024-09-18T01:18:59.522 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130865.log 2024-09-18T01:18:59.522 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101519.log 2024-09-18T01:18:59.523 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107773.log 2024-09-18T01:18:59.523 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141766.log 2024-09-18T01:18:59.523 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127448.log 2024-09-18T01:18:59.524 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31240.log 2024-09-18T01:18:59.524 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79178.log 2024-09-18T01:18:59.524 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47949.log 2024-09-18T01:18:59.525 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43357.log 2024-09-18T01:18:59.525 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42421.log 2024-09-18T01:18:59.525 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74023.log 2024-09-18T01:18:59.526 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90347.log 2024-09-18T01:18:59.526 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128394.log 2024-09-18T01:18:59.526 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130767.log 2024-09-18T01:18:59.527 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130177.log 2024-09-18T01:18:59.527 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50078.log 2024-09-18T01:18:59.528 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97069.log 2024-09-18T01:18:59.528 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115748.log 2024-09-18T01:18:59.528 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54848.log 2024-09-18T01:18:59.529 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65803.log 2024-09-18T01:18:59.529 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121910.log 2024-09-18T01:18:59.529 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132079.log 2024-09-18T01:18:59.529 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38257.log 2024-09-18T01:18:59.530 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113858.log 2024-09-18T01:18:59.530 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93231.log 2024-09-18T01:18:59.530 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81835.log 2024-09-18T01:18:59.531 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113281.log 2024-09-18T01:18:59.531 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55564.log 2024-09-18T01:18:59.531 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76125.log 2024-09-18T01:18:59.532 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99245.log 2024-09-18T01:18:59.532 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59501.log 2024-09-18T01:18:59.533 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136530.log 2024-09-18T01:18:59.533 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46576.log 2024-09-18T01:18:59.533 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72591.log 2024-09-18T01:18:59.533 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89086.log 2024-09-18T01:18:59.534 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60920.log 2024-09-18T01:18:59.534 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123691.log 2024-09-18T01:18:59.534 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58451.log 2024-09-18T01:18:59.535 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58450.log 2024-09-18T01:18:59.535 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49669.log 2024-09-18T01:18:59.536 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44683.log 2024-09-18T01:18:59.536 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69705.log 2024-09-18T01:18:59.536 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136405.log 2024-09-18T01:18:59.537 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135133.log 2024-09-18T01:18:59.537 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62293.log 2024-09-18T01:18:59.537 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70990.log 2024-09-18T01:18:59.538 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41251.log 2024-09-18T01:18:59.538 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52624.log 2024-09-18T01:18:59.538 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75065.log 2024-09-18T01:18:59.539 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75151.log 2024-09-18T01:18:59.539 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127553.log 2024-09-18T01:18:59.539 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46052.log 2024-09-18T01:18:59.540 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115682.log 2024-09-18T01:18:59.540 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67557.log 2024-09-18T01:18:59.540 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50203.log 2024-09-18T01:18:59.540 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103208.log 2024-09-18T01:18:59.541 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101983.log 2024-09-18T01:18:59.541 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60348.log 2024-09-18T01:18:59.542 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128396.log 2024-09-18T01:18:59.542 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123774.log 2024-09-18T01:18:59.543 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81079.log 2024-09-18T01:18:59.543 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86968.log 2024-09-18T01:18:59.543 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68440.log 2024-09-18T01:18:59.544 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83542.log 2024-09-18T01:18:59.544 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133069.log 2024-09-18T01:18:59.545 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79774.log 2024-09-18T01:18:59.545 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122745.log 2024-09-18T01:18:59.545 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37685.log 2024-09-18T01:18:59.546 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44201.log 2024-09-18T01:18:59.546 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42811.log 2024-09-18T01:18:59.546 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48955.log 2024-09-18T01:18:59.547 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30184.log 2024-09-18T01:18:59.547 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124680.log 2024-09-18T01:18:59.547 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120479.log 2024-09-18T01:18:59.547 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112166.log 2024-09-18T01:18:59.547 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31511.log 2024-09-18T01:18:59.548 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118393.log 2024-09-18T01:18:59.548 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45420.log 2024-09-18T01:18:59.549 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65667.log 2024-09-18T01:18:59.549 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116778.log 2024-09-18T01:18:59.549 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.129782.log 2024-09-18T01:18:59.550 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86909.log 2024-09-18T01:18:59.550 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111686.log 2024-09-18T01:18:59.550 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109803.log 2024-09-18T01:18:59.551 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65455.log 2024-09-18T01:18:59.551 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107291.log 2024-09-18T01:18:59.551 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.118092.log 2024-09-18T01:18:59.552 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132500.log 2024-09-18T01:18:59.552 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70293.log 2024-09-18T01:18:59.552 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142338.log 2024-09-18T01:18:59.553 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89714.log 2024-09-18T01:18:59.553 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64544.log 2024-09-18T01:18:59.553 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138083.log 2024-09-18T01:18:59.554 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62743.log 2024-09-18T01:18:59.554 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36317.log 2024-09-18T01:18:59.555 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141974.log 2024-09-18T01:18:59.555 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132648.log 2024-09-18T01:18:59.556 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66945.log 2024-09-18T01:18:59.556 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114274.log 2024-09-18T01:18:59.556 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114699.log 2024-09-18T01:18:59.557 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126360.log 2024-09-18T01:18:59.557 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75349.log 2024-09-18T01:18:59.558 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134457.log 2024-09-18T01:18:59.558 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110227.log 2024-09-18T01:18:59.558 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110448.log 2024-09-18T01:18:59.558 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72894.log 2024-09-18T01:18:59.558 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134207.log 2024-09-18T01:18:59.559 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48946.log 2024-09-18T01:18:59.559 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132055.log 2024-09-18T01:18:59.559 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35199.log 2024-09-18T01:18:59.560 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111205.log 2024-09-18T01:18:59.560 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29737.log 2024-09-18T01:18:59.560 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139776.log 2024-09-18T01:18:59.561 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99541.log 2024-09-18T01:18:59.561 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37367.log 2024-09-18T01:18:59.562 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66676.log 2024-09-18T01:18:59.562 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84122.log 2024-09-18T01:18:59.563 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139428.log 2024-09-18T01:18:59.563 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43356.log 2024-09-18T01:18:59.563 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73708.log 2024-09-18T01:18:59.563 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113641.log 2024-09-18T01:18:59.564 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55436.log 2024-09-18T01:18:59.564 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104049.log 2024-09-18T01:18:59.565 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140706.log 2024-09-18T01:18:59.565 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126863.log 2024-09-18T01:18:59.566 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117596.log 2024-09-18T01:18:59.566 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60136.log 2024-09-18T01:18:59.566 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36570.log 2024-09-18T01:18:59.566 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37472.log 2024-09-18T01:18:59.566 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110741.log 2024-09-18T01:18:59.567 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78085.log 2024-09-18T01:18:59.567 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97847.log 2024-09-18T01:18:59.567 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99160.log 2024-09-18T01:18:59.568 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128171.log 2024-09-18T01:18:59.568 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103503.log 2024-09-18T01:18:59.568 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67807.log 2024-09-18T01:18:59.569 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88285.log 2024-09-18T01:18:59.569 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42980.log 2024-09-18T01:18:59.570 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99457.log 2024-09-18T01:18:59.570 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73256.log 2024-09-18T01:18:59.570 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89508.log 2024-09-18T01:18:59.571 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127452.log 2024-09-18T01:18:59.571 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131253.log 2024-09-18T01:18:59.572 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42844.log 2024-09-18T01:18:59.572 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96841.log 2024-09-18T01:18:59.572 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56134.log 2024-09-18T01:18:59.572 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135529.log 2024-09-18T01:18:59.573 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137663.log 2024-09-18T01:18:59.573 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58825.log 2024-09-18T01:18:59.573 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120383.log 2024-09-18T01:18:59.574 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93292.log 2024-09-18T01:18:59.575 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54094.log 2024-09-18T01:18:59.575 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99883.log 2024-09-18T01:18:59.575 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53893.log 2024-09-18T01:18:59.575 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72933.log 2024-09-18T01:18:59.576 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117691.log 2024-09-18T01:18:59.576 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85847.log 2024-09-18T01:18:59.576 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93084.log 2024-09-18T01:18:59.577 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115897.log 2024-09-18T01:18:59.577 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64329.log 2024-09-18T01:18:59.577 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136823.log 2024-09-18T01:18:59.578 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64401.log 2024-09-18T01:18:59.578 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110425.log 2024-09-18T01:18:59.579 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45314.log 2024-09-18T01:18:59.579 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69288.log 2024-09-18T01:18:59.579 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43247.log 2024-09-18T01:18:59.579 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64198.log 2024-09-18T01:18:59.580 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82084.log 2024-09-18T01:18:59.580 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73019.log 2024-09-18T01:18:59.581 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53554.log 2024-09-18T01:18:59.581 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.140213.log 2024-09-18T01:18:59.581 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92876.log 2024-09-18T01:18:59.582 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35836.log 2024-09-18T01:18:59.582 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136403.log 2024-09-18T01:18:59.582 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96996.log 2024-09-18T01:18:59.583 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69035.log 2024-09-18T01:18:59.583 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62515.log 2024-09-18T01:18:59.583 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126295.log 2024-09-18T01:18:59.584 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86967.log 2024-09-18T01:18:59.584 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62992.log 2024-09-18T01:18:59.585 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124341.log 2024-09-18T01:18:59.585 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31858.log 2024-09-18T01:18:59.586 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99287.log 2024-09-18T01:18:59.586 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61919.log 2024-09-18T01:18:59.586 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78629.log 2024-09-18T01:18:59.586 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39390.log 2024-09-18T01:18:59.587 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84260.log 2024-09-18T01:18:59.587 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77243.log 2024-09-18T01:18:59.587 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100471.log 2024-09-18T01:18:59.588 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100010.log 2024-09-18T01:18:59.588 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93400.log 2024-09-18T01:18:59.589 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36527.log 2024-09-18T01:18:59.589 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125664.log 2024-09-18T01:18:59.589 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50570.log 2024-09-18T01:18:59.590 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89297.log 2024-09-18T01:18:59.590 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59161.log 2024-09-18T01:18:59.590 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137637.log 2024-09-18T01:18:59.591 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120374.log 2024-09-18T01:18:59.591 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131715.log 2024-09-18T01:18:59.592 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.94646.log 2024-09-18T01:18:59.592 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66819.log 2024-09-18T01:18:59.593 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75318.log 2024-09-18T01:18:59.593 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137032.log 2024-09-18T01:18:59.594 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128187.log 2024-09-18T01:18:59.594 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88706.log 2024-09-18T01:18:59.594 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84446.log 2024-09-18T01:18:59.594 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46785.log 2024-09-18T01:18:59.595 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127988.log 2024-09-18T01:18:59.595 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133111.log 2024-09-18T01:18:59.595 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114215.log 2024-09-18T01:18:59.595 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111266.log 2024-09-18T01:18:59.596 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87560.log 2024-09-18T01:18:59.596 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108457.log 2024-09-18T01:18:59.596 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52303.log 2024-09-18T01:18:59.597 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113071.log 2024-09-18T01:18:59.597 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98352.log 2024-09-18T01:18:59.598 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38589.log 2024-09-18T01:18:59.598 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127995.log 2024-09-18T01:18:59.598 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107096.log 2024-09-18T01:18:59.599 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90076.log 2024-09-18T01:18:59.599 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64969.log 2024-09-18T01:18:59.599 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86247.log 2024-09-18T01:18:59.600 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80279.log 2024-09-18T01:18:59.600 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70209.log 2024-09-18T01:18:59.600 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64482.log 2024-09-18T01:18:59.601 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132778.log 2024-09-18T01:18:59.601 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137160.log 2024-09-18T01:18:59.601 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119222.log 2024-09-18T01:18:59.602 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67515.log 2024-09-18T01:18:59.602 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89444.log 2024-09-18T01:18:59.603 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35452.log 2024-09-18T01:18:59.603 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72496.log 2024-09-18T01:18:59.604 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54930.log 2024-09-18T01:18:59.604 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.120548.log 2024-09-18T01:18:59.604 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89234.log 2024-09-18T01:18:59.605 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36780.log 2024-09-18T01:18:59.605 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83834.log 2024-09-18T01:18:59.605 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100679.log 2024-09-18T01:18:59.606 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126781.log 2024-09-18T01:18:59.606 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.98175.log 2024-09-18T01:18:59.606 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29046.log 2024-09-18T01:18:59.607 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88816.log 2024-09-18T01:18:59.607 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69875.log 2024-09-18T01:18:59.608 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95684.log 2024-09-18T01:18:59.608 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42549.log 2024-09-18T01:18:59.609 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40604.log 2024-09-18T01:18:59.609 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61974.log 2024-09-18T01:18:59.610 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64399.log 2024-09-18T01:18:59.610 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97446.log 2024-09-18T01:18:59.611 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37199.log 2024-09-18T01:18:59.611 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39525.log 2024-09-18T01:18:59.611 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111101.log 2024-09-18T01:18:59.612 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78840.log 2024-09-18T01:18:59.612 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72601.log 2024-09-18T01:18:59.612 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90556.log 2024-09-18T01:18:59.613 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83008.log 2024-09-18T01:18:59.613 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57986.log 2024-09-18T01:18:59.613 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105063.log 2024-09-18T01:18:59.613 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61773.log 2024-09-18T01:18:59.615 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81538.log 2024-09-18T01:18:59.617 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84706.log 2024-09-18T01:18:59.618 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64361.log 2024-09-18T01:18:59.618 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81118.log 2024-09-18T01:18:59.618 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79345.log 2024-09-18T01:18:59.618 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39695.log 2024-09-18T01:18:59.618 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67765.log 2024-09-18T01:18:59.618 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96882.log 2024-09-18T01:18:59.618 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129855.log 2024-09-18T01:18:59.618 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117315.log 2024-09-18T01:18:59.618 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77624.log 2024-09-18T01:18:59.618 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62306.log 2024-09-18T01:18:59.620 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81414.log 2024-09-18T01:18:59.620 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99638.log 2024-09-18T01:18:59.620 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122240.log 2024-09-18T01:18:59.620 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135559.log 2024-09-18T01:18:59.620 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60658.log 2024-09-18T01:18:59.620 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110544.log 2024-09-18T01:18:59.621 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102450.log 2024-09-18T01:18:59.621 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31023.log 2024-09-18T01:18:59.621 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39372.log 2024-09-18T01:18:59.621 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107171.log 2024-09-18T01:18:59.622 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47990.log 2024-09-18T01:18:59.622 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80951.log 2024-09-18T01:18:59.623 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95949.log 2024-09-18T01:18:59.623 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132603.log 2024-09-18T01:18:59.623 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129283.log 2024-09-18T01:18:59.624 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87918.log 2024-09-18T01:18:59.624 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84556.log 2024-09-18T01:18:59.624 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118973.log 2024-09-18T01:18:59.624 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140654.log 2024-09-18T01:18:59.625 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31061.log 2024-09-18T01:18:59.625 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81959.log 2024-09-18T01:18:59.626 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80407.log 2024-09-18T01:18:59.626 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137453.log 2024-09-18T01:18:59.626 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85391.log 2024-09-18T01:18:59.626 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49888.log 2024-09-18T01:18:59.627 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41913.log 2024-09-18T01:18:59.627 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.127621.log 2024-09-18T01:18:59.628 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80068.log 2024-09-18T01:18:59.628 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126106.log 2024-09-18T01:18:59.629 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140686.log 2024-09-18T01:18:59.629 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51569.log 2024-09-18T01:18:59.629 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40127.log 2024-09-18T01:18:59.630 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123211.log 2024-09-18T01:18:59.630 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139232.log 2024-09-18T01:18:59.631 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104343.log 2024-09-18T01:18:59.631 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126235.log 2024-09-18T01:18:59.631 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121700.log 2024-09-18T01:18:59.631 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138002.log 2024-09-18T01:18:59.632 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93875.log 2024-09-18T01:18:59.632 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125897.log 2024-09-18T01:18:59.633 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108135.log 2024-09-18T01:18:59.633 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42240.log 2024-09-18T01:18:59.633 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34215.log 2024-09-18T01:18:59.634 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105442.log 2024-09-18T01:18:59.634 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50710.log 2024-09-18T01:18:59.634 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126210.log 2024-09-18T01:18:59.635 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136110.log 2024-09-18T01:18:59.635 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32577.log 2024-09-18T01:18:59.635 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105471.log 2024-09-18T01:18:59.636 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58785.log 2024-09-18T01:18:59.636 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60502.log 2024-09-18T01:18:59.637 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100514.log 2024-09-18T01:18:59.637 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79859.log 2024-09-18T01:18:59.637 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32681.log 2024-09-18T01:18:59.638 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114970.log 2024-09-18T01:18:59.638 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93779.log 2024-09-18T01:18:59.638 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2024-09-18T01:18:59.639 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102787.log 2024-09-18T01:18:59.639 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55059.log 2024-09-18T01:18:59.639 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117698.log 2024-09-18T01:18:59.640 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32268.log 2024-09-18T01:18:59.641 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109829.log 2024-09-18T01:18:59.641 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84172.log 2024-09-18T01:18:59.641 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82618.log 2024-09-18T01:18:59.641 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62422.log 2024-09-18T01:18:59.642 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97224.log 2024-09-18T01:18:59.642 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72228.log 2024-09-18T01:18:59.642 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70843.log 2024-09-18T01:18:59.643 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140683.log 2024-09-18T01:18:59.643 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79387.log 2024-09-18T01:18:59.644 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40470.log 2024-09-18T01:18:59.644 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127158.log 2024-09-18T01:18:59.644 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115853.log 2024-09-18T01:18:59.645 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.139561.log 2024-09-18T01:18:59.645 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79094.log 2024-09-18T01:18:59.646 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78295.log 2024-09-18T01:18:59.646 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37641.log 2024-09-18T01:18:59.646 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65414.log 2024-09-18T01:18:59.647 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67096.log 2024-09-18T01:18:59.647 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130083.log 2024-09-18T01:18:59.647 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92032.log 2024-09-18T01:18:59.648 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58239.log 2024-09-18T01:18:59.648 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135342.log 2024-09-18T01:18:59.648 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35098.log 2024-09-18T01:18:59.649 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68017.log 2024-09-18T01:18:59.649 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69832.log 2024-09-18T01:18:59.649 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51776.log 2024-09-18T01:18:59.650 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103083.log 2024-09-18T01:18:59.651 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124470.log 2024-09-18T01:18:59.651 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37052.log 2024-09-18T01:18:59.651 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38585.log 2024-09-18T01:18:59.651 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38548.log 2024-09-18T01:18:59.652 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126505.log 2024-09-18T01:18:59.653 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91616.log 2024-09-18T01:18:59.653 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56006.log 2024-09-18T01:18:59.653 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108390.log 2024-09-18T01:18:59.668 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133027.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36462.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67895.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104134.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127115.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140748.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80532.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104513.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75064.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38799.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91844.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73016.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62556.log 2024-09-18T01:18:59.669 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46827.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105572.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.106742.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121258.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67348.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141401.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58089.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33164.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36950.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135361.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116218.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70419.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123479.log 2024-09-18T01:18:59.670 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43784.log 2024-09-18T01:18:59.671 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33417.log 2024-09-18T01:18:59.671 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40376.log 2024-09-18T01:18:59.671 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41231.log 2024-09-18T01:18:59.671 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116215.log 2024-09-18T01:18:59.671 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44834.log 2024-09-18T01:18:59.673 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103375.log 2024-09-18T01:18:59.673 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136402.log 2024-09-18T01:18:59.673 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68313.log 2024-09-18T01:18:59.673 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43993.log 2024-09-18T01:18:59.673 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32745.log 2024-09-18T01:18:59.673 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33374.log 2024-09-18T01:18:59.673 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.131132.log 2024-09-18T01:18:59.673 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56089.log 2024-09-18T01:18:59.673 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92394.log 2024-09-18T01:18:59.673 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30015.log 2024-09-18T01:18:59.674 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67179.log 2024-09-18T01:18:59.675 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76193.log 2024-09-18T01:18:59.675 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118725.log 2024-09-18T01:18:59.675 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128165.log 2024-09-18T01:18:59.675 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103628.log 2024-09-18T01:18:59.675 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83897.log 2024-09-18T01:18:59.676 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123848.log 2024-09-18T01:18:59.676 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133029.log 2024-09-18T01:18:59.676 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30869.log 2024-09-18T01:18:59.677 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114632.log 2024-09-18T01:18:59.677 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135513.log 2024-09-18T01:18:59.677 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90221.log 2024-09-18T01:18:59.680 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44833.log 2024-09-18T01:18:59.680 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130694.log 2024-09-18T01:18:59.680 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126150.log 2024-09-18T01:18:59.680 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55837.log 2024-09-18T01:18:59.680 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115474.log 2024-09-18T01:18:59.680 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122934.log 2024-09-18T01:18:59.681 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121190.log 2024-09-18T01:18:59.681 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33730.log 2024-09-18T01:18:59.681 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134297.log 2024-09-18T01:18:59.682 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36085.log 2024-09-18T01:18:59.682 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.107035.log 2024-09-18T01:18:59.683 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116787.log 2024-09-18T01:18:59.683 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40283.log 2024-09-18T01:18:59.683 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.94094.log 2024-09-18T01:18:59.684 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128759.log 2024-09-18T01:18:59.684 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117704.log 2024-09-18T01:18:59.684 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36885.log 2024-09-18T01:18:59.685 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30752.log 2024-09-18T01:18:59.685 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134079.log 2024-09-18T01:18:59.686 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55922.log 2024-09-18T01:18:59.686 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87474.log 2024-09-18T01:18:59.686 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125789.log 2024-09-18T01:18:59.687 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30830.log 2024-09-18T01:18:59.687 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107128.log 2024-09-18T01:18:59.687 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38937.log 2024-09-18T01:18:59.688 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101404.log 2024-09-18T01:18:59.688 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37368.log 2024-09-18T01:18:59.688 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.134647.log 2024-09-18T01:18:59.689 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47359.log 2024-09-18T01:18:59.689 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64636.log 2024-09-18T01:18:59.689 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37516.log 2024-09-18T01:18:59.690 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114843.log 2024-09-18T01:18:59.691 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80740.log 2024-09-18T01:18:59.691 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103883.log 2024-09-18T01:18:59.691 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68737.log 2024-09-18T01:18:59.692 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66925.log 2024-09-18T01:18:59.692 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43994.log 2024-09-18T01:18:59.693 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83712.log 2024-09-18T01:18:59.693 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44958.log 2024-09-18T01:18:59.694 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124679.log 2024-09-18T01:18:59.694 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116757.log 2024-09-18T01:18:59.694 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33059.log 2024-09-18T01:18:59.695 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111204.log 2024-09-18T01:18:59.695 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120422.log 2024-09-18T01:18:59.695 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31448.log 2024-09-18T01:18:59.696 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135899.log 2024-09-18T01:18:59.696 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42633.log 2024-09-18T01:18:59.697 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129530.log 2024-09-18T01:18:59.697 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62284.log 2024-09-18T01:18:59.697 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91359.log 2024-09-18T01:18:59.698 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.116731.log 2024-09-18T01:18:59.698 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102065.log 2024-09-18T01:18:59.699 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76896.log 2024-09-18T01:18:59.699 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111810.log 2024-09-18T01:18:59.700 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142390.log 2024-09-18T01:18:59.700 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99840.log 2024-09-18T01:18:59.700 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59581.log 2024-09-18T01:18:59.701 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38256.log 2024-09-18T01:18:59.701 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88329.log 2024-09-18T01:18:59.701 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63794.log 2024-09-18T01:18:59.702 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136780.log 2024-09-18T01:18:59.702 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137370.log 2024-09-18T01:18:59.703 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132824.log 2024-09-18T01:18:59.703 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65328.log 2024-09-18T01:18:59.703 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117693.log 2024-09-18T01:18:59.704 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99050.log 2024-09-18T01:18:59.704 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65373.log 2024-09-18T01:18:59.705 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131716.log 2024-09-18T01:18:59.708 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61305.log 2024-09-18T01:18:59.708 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111682.log 2024-09-18T01:18:59.708 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65715.log 2024-09-18T01:18:59.708 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62155.log 2024-09-18T01:18:59.708 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39384.log 2024-09-18T01:18:59.708 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129265.log 2024-09-18T01:18:59.708 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42422.log 2024-09-18T01:18:59.710 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49356.log 2024-09-18T01:18:59.710 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75472.log 2024-09-18T01:18:59.710 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40066.log 2024-09-18T01:18:59.710 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34888.log 2024-09-18T01:18:59.710 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39474.log 2024-09-18T01:18:59.711 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123901.log 2024-09-18T01:18:59.711 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43950.log 2024-09-18T01:18:59.711 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80573.log 2024-09-18T01:18:59.712 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40558.log 2024-09-18T01:18:59.712 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33837.log 2024-09-18T01:18:59.712 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61353.log 2024-09-18T01:18:59.712 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130311.log 2024-09-18T01:18:59.720 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117883.log 2024-09-18T01:18:59.720 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36423.log 2024-09-18T01:18:59.720 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102743.log 2024-09-18T01:18:59.720 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92601.log 2024-09-18T01:18:59.720 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79944.log 2024-09-18T01:18:59.720 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81370.log 2024-09-18T01:18:59.720 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126780.log 2024-09-18T01:18:59.721 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86704.log 2024-09-18T01:18:59.721 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79093.log 2024-09-18T01:18:59.721 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116522.log 2024-09-18T01:18:59.721 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110355.log 2024-09-18T01:18:59.721 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65843.log 2024-09-18T01:18:59.721 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84896.log 2024-09-18T01:18:59.721 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54403.log 2024-09-18T01:18:59.721 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137452.log 2024-09-18T01:18:59.723 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42114.log 2024-09-18T01:18:59.723 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56679.log 2024-09-18T01:18:59.723 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100617.log 2024-09-18T01:18:59.723 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85303.log 2024-09-18T01:18:59.723 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36672.log 2024-09-18T01:18:59.723 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49113.log 2024-09-18T01:18:59.723 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.142041.log 2024-09-18T01:18:59.725 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111034.log 2024-09-18T01:18:59.725 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59455.log 2024-09-18T01:18:59.725 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68865.log 2024-09-18T01:18:59.725 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.109891.log 2024-09-18T01:18:59.725 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87759.log 2024-09-18T01:18:59.726 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91404.log 2024-09-18T01:18:59.726 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76636.log 2024-09-18T01:18:59.726 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83763.log 2024-09-18T01:18:59.726 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80824.log 2024-09-18T01:18:59.727 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82190.log 2024-09-18T01:18:59.727 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50741.log 2024-09-18T01:18:59.728 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92666.log 2024-09-18T01:18:59.728 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88129.log 2024-09-18T01:18:59.728 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67681.log 2024-09-18T01:18:59.729 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56425.log 2024-09-18T01:18:59.729 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68019.log 2024-09-18T01:18:59.729 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71388.log 2024-09-18T01:18:59.730 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102320.log 2024-09-18T01:18:59.730 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111328.log 2024-09-18T01:18:59.731 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42204.log 2024-09-18T01:18:59.731 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75880.log 2024-09-18T01:18:59.732 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87247.log 2024-09-18T01:18:59.732 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56424.log 2024-09-18T01:18:59.732 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.118672.log 2024-09-18T01:18:59.733 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54763.log 2024-09-18T01:18:59.733 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87225.log 2024-09-18T01:18:59.734 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49281.log 2024-09-18T01:18:59.734 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43417.log 2024-09-18T01:18:59.734 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.141649.log 2024-09-18T01:18:59.735 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68501.log 2024-09-18T01:18:59.735 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64684.log 2024-09-18T01:18:59.736 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104301.log 2024-09-18T01:18:59.736 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113789.log 2024-09-18T01:18:59.736 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39985.log 2024-09-18T01:18:59.737 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.118080.log 2024-09-18T01:18:59.737 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114440.log 2024-09-18T01:18:59.738 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38393.log 2024-09-18T01:18:59.738 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74304.log 2024-09-18T01:18:59.738 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52356.log 2024-09-18T01:18:59.739 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122242.log 2024-09-18T01:18:59.739 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65587.log 2024-09-18T01:18:59.740 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61020.log 2024-09-18T01:18:59.740 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129310.log 2024-09-18T01:18:59.741 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102874.log 2024-09-18T01:18:59.741 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62951.log 2024-09-18T01:18:59.741 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67599.log 2024-09-18T01:18:59.742 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124425.log 2024-09-18T01:18:59.742 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51320.log 2024-09-18T01:18:59.742 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115898.log 2024-09-18T01:18:59.743 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91552.log 2024-09-18T01:18:59.743 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138515.log 2024-09-18T01:18:59.744 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99926.log 2024-09-18T01:18:59.744 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53105.log 2024-09-18T01:18:59.745 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51247.log 2024-09-18T01:18:59.745 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111956.log 2024-09-18T01:18:59.745 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129207.log 2024-09-18T01:18:59.746 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61496.log 2024-09-18T01:18:59.746 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76436.log 2024-09-18T01:18:59.747 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32932.log 2024-09-18T01:18:59.747 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89169.log 2024-09-18T01:18:59.747 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39397.log 2024-09-18T01:18:59.748 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85209.log 2024-09-18T01:18:59.748 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.105072.log 2024-09-18T01:18:59.749 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104468.log 2024-09-18T01:18:59.749 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.128312.log 2024-09-18T01:18:59.749 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67471.log 2024-09-18T01:18:59.750 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.97397.log 2024-09-18T01:18:59.750 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109396.log 2024-09-18T01:18:59.751 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59710.log 2024-09-18T01:18:59.751 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66057.log 2024-09-18T01:18:59.752 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87072.log 2024-09-18T01:18:59.752 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50321.log 2024-09-18T01:18:59.753 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125896.log 2024-09-18T01:18:59.753 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2024-09-18T01:18:59.753 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.116208.log 2024-09-18T01:18:59.754 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32954.log 2024-09-18T01:18:59.754 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75451.log 2024-09-18T01:18:59.755 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58870.log 2024-09-18T01:18:59.755 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108583.log 2024-09-18T01:18:59.755 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47841.log 2024-09-18T01:18:59.756 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.106741.log 2024-09-18T01:18:59.756 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87420.log 2024-09-18T01:18:59.757 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82168.log 2024-09-18T01:18:59.757 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.110101.log 2024-09-18T01:18:59.758 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135900.log 2024-09-18T01:18:59.758 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83706.log 2024-09-18T01:18:59.758 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59415.log 2024-09-18T01:18:59.759 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129262.log 2024-09-18T01:18:59.759 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137874.log 2024-09-18T01:18:59.760 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111541.log 2024-09-18T01:18:59.760 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.128963.log 2024-09-18T01:18:59.761 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47906.log 2024-09-18T01:18:59.761 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.117702.log 2024-09-18T01:18:59.761 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92663.log 2024-09-18T01:18:59.762 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76645.log 2024-09-18T01:18:59.762 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.111957.log 2024-09-18T01:18:59.763 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127264.log 2024-09-18T01:18:59.763 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130405.log 2024-09-18T01:18:59.764 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103417.log 2024-09-18T01:18:59.764 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62495.log 2024-09-18T01:18:59.764 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67053.log 2024-09-18T01:18:59.765 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81245.log 2024-09-18T01:18:59.765 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142156.log 2024-09-18T01:18:59.766 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32996.log 2024-09-18T01:18:59.766 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37412.log 2024-09-18T01:18:59.766 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109765.log 2024-09-18T01:18:59.767 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74878.log 2024-09-18T01:18:59.767 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.133742.log 2024-09-18T01:18:59.768 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140924.log 2024-09-18T01:18:59.768 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69874.log 2024-09-18T01:18:59.769 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94517.log 2024-09-18T01:18:59.769 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45211.log 2024-09-18T01:18:59.769 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100133.log 2024-09-18T01:18:59.770 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91109.log 2024-09-18T01:18:59.770 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31308.log 2024-09-18T01:18:59.771 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82423.log 2024-09-18T01:18:59.771 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110950.log 2024-09-18T01:18:59.772 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96453.log 2024-09-18T01:18:59.772 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79816.log 2024-09-18T01:18:59.772 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.129594.log 2024-09-18T01:18:59.773 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86007.log 2024-09-18T01:18:59.773 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.130607.log 2024-09-18T01:18:59.774 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29246.log 2024-09-18T01:18:59.774 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99330.log 2024-09-18T01:18:59.774 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.140459.log 2024-09-18T01:18:59.775 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112732.log 2024-09-18T01:18:59.775 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.118438.log 2024-09-18T01:18:59.776 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99968.log 2024-09-18T01:18:59.776 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57392.log 2024-09-18T01:18:59.777 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54319.log 2024-09-18T01:18:59.777 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73673.log 2024-09-18T01:18:59.777 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74837.log 2024-09-18T01:18:59.778 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66099.log 2024-09-18T01:18:59.778 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122451.log 2024-09-18T01:18:59.779 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70544.log 2024-09-18T01:18:59.779 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.117053.log 2024-09-18T01:18:59.780 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114277.log 2024-09-18T01:18:59.780 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57032.log 2024-09-18T01:18:59.781 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68101.log 2024-09-18T01:18:59.781 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78219.log 2024-09-18T01:18:59.781 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131428.log 2024-09-18T01:18:59.782 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137580.log 2024-09-18T01:18:59.782 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107866.log 2024-09-18T01:18:59.783 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.108276.log 2024-09-18T01:18:59.783 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66014.log 2024-09-18T01:18:59.784 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119847.log 2024-09-18T01:18:59.784 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41492.log 2024-09-18T01:18:59.784 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29823.log 2024-09-18T01:18:59.785 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109491.log 2024-09-18T01:18:59.785 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73063.log 2024-09-18T01:18:59.786 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81076.log 2024-09-18T01:18:59.786 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61677.log 2024-09-18T01:18:59.787 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131228.log 2024-09-18T01:18:59.787 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44894.log 2024-09-18T01:18:59.787 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115327.log 2024-09-18T01:18:59.788 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77833.log 2024-09-18T01:18:59.788 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75203.log 2024-09-18T01:18:59.788 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50257.log 2024-09-18T01:18:59.789 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93851.log 2024-09-18T01:18:59.789 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56845.log 2024-09-18T01:18:59.789 INFO:teuthology.orchestra.run.smithi049.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108621.log 2024-09-18T01:19:00.978 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.62764.log: /var/log/ceph/cluster1-client.mirror.121085.log: /var/log/ceph/cluster1-client.mirror.102023.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.84500.log.gz/var/log/ceph/cluster1-client.admin.118699.log: 2024-09-18T01:19:00.978 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.108302.log: /var/log/ceph/cluster1-client.mirror.40967.log: /var/log/ceph/cluster2-client.mirror.107980.log: /var/log/ceph/cluster2-client.admin.139430.log: /var/log/ceph/cluster2-client.admin.105223.log: /var/log/ceph/cluster1-client.mirror.59456.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.139430.log.gz 2024-09-18T01:19:00.978 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89237.log: /var/log/ceph/cluster1-client.mirror.115117.log: /var/log/ceph/cluster1-client.mirror.133155.log: /var/log/ceph/cluster2-client.admin.129471.log: /var/log/ceph/cluster1-client.admin.72120.log: /var/log/ceph/cluster1-client.admin.116258.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.72721.log.gz 2024-09-18T01:19:00.978 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.141079.log: /var/log/ceph/cluster1-client.mirror.87753.log: /var/log/ceph/cluster1-client.mirror.34049.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.72752.log.gz 2024-09-18T01:19:00.978 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.79563.log: /var/log/ceph/cluster1-client.mirror.137034.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.116258.log.gz 2024-09-18T01:19:00.978 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.64763.log: /var/log/ceph/cluster1-client.admin.42319.log: /var/log/ceph/cluster1-client.mirror.32069.log: /var/log/ceph/cluster1-client.admin.82935.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.73304.log.gz 2024-09-18T01:19:00.982 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113010.log: /var/log/ceph/cluster1-client.admin.86236.log: /var/log/ceph/cluster1-client.mirror.102742.log: /var/log/ceph/cluster1-client.mirror.30580.log: /var/log/ceph/cluster1-client.mirror.90348.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.141079.log.gz/var/log/ceph/cluster1-client.mirror.132750.log: 2024-09-18T01:19:00.982 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.51455.log: /var/log/ceph/cluster1-client.mirror.105479.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.86236.log.gz 2024-09-18T01:19:00.982 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93356.log: /var/log/ceph/cluster1-client.admin.128179.log: /var/log/ceph/cluster1-client.mirror.59711.log: /var/log/ceph/cluster1-client.mirror.69918.log: /var/log/ceph/cluster1-client.mirror.90896.log: /var/log/ceph/cluster1-client.admin.130265.log: /var/log/ceph/cluster2-client.mirror.49803.log: /var/log/ceph/cluster1-client.admin.107364.log: /var/log/ceph/cluster1-client.mirror.111161.log: /var/log/ceph/cluster2-client.admin.140480.log: /var/log/ceph/cluster1-client.mirror.85458.log: /var/log/ceph/cluster1-client.mirror.134139.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47993.log.gz 2024-09-18T01:19:00.982 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91319.log: /var/log/ceph/cluster2-client.mirror.140193.log: /var/log/ceph/cluster1-client.mirror.57732.log: /var/log/ceph/cluster1-client.mirror.34635.log: /var/log/ceph/cluster1-client.mirror.89801.log: /var/log/ceph/cluster1-client.mirror.53488.log: /var/log/ceph/cluster1-client.mirror.67305.log: /var/log/ceph/cluster1-client.mirror.81037.log: /var/log/ceph/cluster1-client.mirror.126316.log: /var/log/ceph/cluster2-client.mirror.62971.log: /var/log/ceph/cluster2-client.admin.118180.log: /var/log/ceph/cluster1-client.admin.83291.log: /var/log/ceph/cluster2-client.admin.129259.log: /var/log/ceph/cluster1-client.mirror.110405.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.106347.log.gz 2024-09-18T01:19:00.982 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123924.log: /var/log/ceph/cluster2-client.mirror.96955.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.83291.log.gz 2024-09-18T01:19:00.982 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129259.log.gz 2024-09-18T01:19:00.982 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67347.log: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.106368.log.gz 2024-09-18T01:19:04.611 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129085.log: /var/log/ceph/cluster1-client.mirror.42265.log: /var/log/ceph/cluster1-client.mirror.58995.log: /var/log/ceph/cluster1-client.mirror.37262.log: /var/log/ceph/cluster1-client.mirror.102043.log: /var/log/ceph/cluster1-client.mirror.99839.log: /var/log/ceph/cluster2-client.mirror.51434.log: /var/log/ceph/cluster1-client.mirror.64859.log: /var/log/ceph/cluster1-client.mirror.123292.log: /var/log/ceph/cluster1-client.admin.132223.log: /var/log/ceph/cluster1-client.mirror.71583.log: /var/log/ceph/cluster2-client.admin.96178.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.116350.log.gz 2024-09-18T01:19:04.611 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.53618.log: /var/log/ceph/cluster1-client.mirror.127286.log: /var/log/ceph/cluster1-client.mirror.135090.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96178.log.gz 2024-09-18T01:19:04.612 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.25267.log: /var/log/ceph/cluster1-client.mirror.111076.log: /var/log/ceph/cluster1-client.mirror.104006.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25267.log.gz 2024-09-18T01:19:04.612 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.129762.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.128169.log.gz 2024-09-18T01:19:04.612 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90852.log: /var/log/ceph/cluster1-client.admin.51931.log: /var/log/ceph/cluster1-client.admin.105097.log: /var/log/ceph/cluster2-client.admin.62314.log: /var/log/ceph/cluster1-client.mirror.75895.log: /var/log/ceph/cluster1-client.mirror.120699.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.105097.log.gz 2024-09-18T01:19:04.621 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.71837.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62314.log.gz 2024-09-18T01:19:04.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135984.log: /var/log/ceph/cluster1-client.mirror.121543.log: /var/log/ceph/cluster2-client.admin.62245.log: /var/log/ceph/cluster1-client.mirror.65714.log: /var/log/ceph/cluster1-client.mirror.89632.log: /var/log/ceph/cluster1-client.mirror.102830.log: /var/log/ceph/cluster1-client.mirror.47486.log: /var/log/ceph/cluster1-client.admin.51589.log: /var/log/ceph/cluster2-client.admin.128491.log: /var/log/ceph/cluster2-client.admin.128710.log: /var/log/ceph/cluster1-client.mirror.101604.log: /var/log/ceph/cluster1-client.mirror.112649.log: /var/log/ceph/cluster1-client.admin.98525.log: /var/log/ceph/cluster1-client.admin.96225.log: /var/log/ceph/cluster2-client.admin.140350.log: /var/log/ceph/cluster1-client.mirror.113366.log: /var/log/ceph/cluster1-client.mirror.101646.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.98525.log.gz 2024-09-18T01:19:04.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.118953.log: /var/log/ceph/cluster1-client.admin.49283.log: /var/log/ceph/cluster1-client.mirror.102278.log: /var/log/ceph/cluster1-client.admin.71246.log: /var/log/ceph/cluster1-client.mirror.129876.log: /var/log/ceph/cluster2-client.admin.50902.log: /var/log/ceph/cluster1-client.mirror.54486.log: /var/log/ceph/cluster2-client.admin.117388.log: /var/log/ceph/cluster1-client.mirror.112377.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.71246.log.gz 2024-09-18T01:19:04.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.98998.log: /var/log/ceph/cluster1-client.mirror.91907.log: /var/log/ceph/cluster1-client.admin.129695.log: /var/log/ceph/cluster1-client.admin.82556.log: /var/log/ceph/cluster1-client.mirror.71034.log: /var/log/ceph/cluster1-client.mirror.65542.log: /var/log/ceph/cluster2-client.mirror.118314.log: /var/log/ceph/cluster2-client.admin.95952.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.82556.log.gz 2024-09-18T01:19:04.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113155.log: /var/log/ceph/cluster1-client.mirror.30476.log: /var/log/ceph/cluster2-client.admin.73677.log: /var/log/ceph/cluster1-client.admin.75650.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.95952.log.gz 2024-09-18T01:19:04.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136821.log: /var/log/ceph/cluster1-client.mirror.56888.log: /var/log/ceph/cluster1-client.admin.84035.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73677.log.gz 2024-09-18T01:19:04.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109176.log: /var/log/ceph/cluster1-client.admin.84501.log: /var/log/ceph/cluster1-client.mirror.75522.log: /var/log/ceph/cluster1-client.mirror.74930.log: /var/log/ceph/cluster1-client.admin.61915.log: /var/log/ceph/cluster2-client.admin.128713.log: /var/log/ceph/cluster1-client.mirror.133365.log: /var/log/ceph/cluster1-client.admin.141334.log: /var/log/ceph/cluster1-client.admin.74278.log: /var/log/ceph/cluster2-client.admin.128711.log: /var/log/ceph/cluster1-client.admin.63536.log: /var/log/ceph/cluster1-client.mirror.130952.log: /var/log/ceph/cluster1-client.mirror.79217.log: /var/log/ceph/cluster1-client.mirror.102234.log: /var/log/ceph/cluster1-client.admin.118088.log: /var/log/ceph/cluster1-client.admin.119269.log: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.50891.log.gz 2024-09-18T01:19:04.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111476.log: /var/log/ceph/cluster1-client.admin.62811.log: /var/log/ceph/cluster1-client.admin.95128.log: /var/log/ceph/cluster2-client.admin.107301.log: /var/log/ceph/cluster1-client.admin.60110.log: /var/log/ceph/cluster1-client.admin.84935.log: /var/log/ceph/cluster1-client.mirror.57265.log: /var/log/ceph/cluster1-client.mirror.32464.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107301.log.gz 2024-09-18T01:19:04.622 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60110.log.gz 2024-09-18T01:19:04.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46094.log: /var/log/ceph/cluster2-client.admin.72578.log: /var/log/ceph/cluster2-client.admin.142182.log: /var/log/ceph/cluster1-client.mirror.104218.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142182.log.gz 2024-09-18T01:19:04.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138252.log: /var/log/ceph/cluster1-client.mirror.96748.log: /var/log/ceph/cluster1-client.admin.38246.log: /var/log/ceph/cluster1-client.mirror.81748.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.38246.log.gz 2024-09-18T01:19:04.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42918.log: /var/log/ceph/cluster1-client.mirror.132980.log: /var/log/ceph/cluster1-client.mirror.114906.log: /var/log/ceph/cluster1-client.mirror.135940.log: /var/log/ceph/cluster1-client.admin.87279.log: /var/log/ceph/cluster1-client.admin.31750.log: /var/log/ceph/cluster2-client.admin.129395.log: /var/log/ceph/cluster1-client.mirror.81244.log: /var/log/ceph/cluster2-client.admin.117991.log: /var/log/ceph/cluster1-client.admin.63039.log: /var/log/ceph/cluster2-client.admin.139467.log: /var/log/ceph/cluster1-client.mirror.77414.log: /var/log/ceph/cluster1-client.mirror.123502.log: /var/log/ceph/cluster1-client.mirror.126736.log: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.141334.log.gz 2024-09-18T01:19:04.622 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33206.log: /var/log/ceph/cluster1-client.admin.141448.log: /var/log/ceph/cluster1-client.admin.131134.log: /var/log/ceph/cluster2-client.admin.118434.log: /var/log/ceph/cluster1-client.mirror.37135.log: /var/log/ceph/cluster2-client.admin.95236.log: /var/log/ceph/cluster1-client.mirror.114760.log: /var/log/ceph/cluster1-client.mirror.71293.log: /var/log/ceph/cluster1-client.admin.64207.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118434.log.gz 2024-09-18T01:19:04.626 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.113184.log: /var/log/ceph/cluster1-client.admin.116349.log: /var/log/ceph/cluster1-client.mirror.77046.log: /var/log/ceph/cluster1-client.mirror.109850.log: /var/log/ceph/cluster1-client.admin.42696.log: /var/log/ceph/cluster1-client.mirror.121635.log: /var/log/ceph/cluster1-client.admin.109512.log: /var/log/ceph/cluster1-client.admin.141676.log: /var/log/ceph/cluster1-client.mirror.64502.log: 91.5%/var/log/ceph/cluster2-client.mirror.118341.log: -- replaced with /var/log/ceph/cluster1-client.admin.49360.log.gz 2024-09-18T01:19:04.626 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.40030.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.109512.log.gz 2024-09-18T01:19:04.626 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106174.log: /var/log/ceph/cluster1-client.mirror.91697.log: /var/log/ceph/cluster1-client.mirror.112798.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.40030.log.gz 2024-09-18T01:19:04.626 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66842.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.106174.log.gz 2024-09-18T01:19:04.626 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.139086.log: /var/log/ceph/cluster1-client.mirror.132385.log: /var/log/ceph/cluster1-client.admin.108966.log: /var/log/ceph/cluster1-client.mirror.134140.log: /var/log/ceph/cluster1-client.mirror.43698.log: /var/log/ceph/cluster1-client.mirror.75374.log: /var/log/ceph/cluster1-client.mirror.76320.log: /var/log/ceph/cluster1-client.mirror.76814.log: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.141448.log.gz 2024-09-18T01:19:04.626 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.134924.log: /var/log/ceph/cluster1-client.mirror.66631.log: /var/log/ceph/cluster1-client.mirror.79136.log: /var/log/ceph/cluster2-client.admin.118528.log: 91.0% -- replaced with /var/log/ceph/cluster1-client.admin.141676.log.gz 2024-09-18T01:19:04.626 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88372.log: /var/log/ceph/cluster1-client.mirror.125454.log: /var/log/ceph/cluster2-client.mirror.85551.log: /var/log/ceph/cluster1-client.mirror.114908.log: /var/log/ceph/cluster1-client.admin.94806.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118528.log.gz 2024-09-18T01:19:04.626 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110990.log: /var/log/ceph/cluster1-client.mirror.90714.log: /var/log/ceph/cluster1-client.mirror.55268.log: /var/log/ceph/cluster1-client.mirror.70253.log: /var/log/ceph/cluster1-client.mirror.46154.log: /var/log/ceph/cluster1-client.mirror.137455.log: /var/log/ceph/cluster1-client.mirror.35684.log: /var/log/ceph/cluster2-client.admin.50318.log: /var/log/ceph/cluster1-client.mirror.125685.log: /var/log/ceph/cluster1-client.mirror.135135.log: /var/log/ceph/cluster2-client.admin.118514.log: /var/log/ceph/cluster1-client.admin.138930.log: /var/log/ceph/cluster1-client.mirror.57097.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50318.log.gz 2024-09-18T01:19:04.626 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.107793.log: /var/log/ceph/cluster2-client.admin.28534.log: /var/log/ceph/cluster1-client.mirror.134296.log: 0.0% 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118514.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.138930.log.gz 2024-09-18T01:19:04.626 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:04.627 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28534.log.gz 2024-09-18T01:19:04.632 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101143.log: /var/log/ceph/cluster1-client.mirror.62878.log: /var/log/ceph/cluster1-client.mirror.110865.log: /var/log/ceph/cluster1-client.admin.49409.log: /var/log/ceph/cluster1-client.mirror.81876.log: /var/log/ceph/cluster2-client.admin.73104.log: /var/log/ceph/cluster1-client.mirror.141287.log: /var/log/ceph/cluster1-client.mirror.37202.log: /var/log/ceph/cluster1-client.mirror.111749.log: /var/log/ceph/cluster1-client.mirror.44747.log: /var/log/ceph/cluster1-client.mirror.79776.log: 67.5% -- replaced with /var/log/ceph/cluster1-client.mirror.141287.log.gz 2024-09-18T01:19:04.632 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.73653.log: /var/log/ceph/cluster1-client.mirror.33520.log: /var/log/ceph/cluster2-client.mirror.139337.log: /var/log/ceph/cluster1-client.admin.63153.log: /var/log/ceph/cluster1-client.mirror.93807.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73653.log.gz 2024-09-18T01:19:04.633 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93210.log: /var/log/ceph/cluster1-client.mirror.66444.log: /var/log/ceph/cluster1-client.admin.53616.log: /var/log/ceph/cluster1-client.mirror.111389.log: /var/log/ceph/cluster2-client.admin.61630.log: /var/log/ceph/cluster2-client.mirror.107504.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.53616.log.gz 2024-09-18T01:19:04.633 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.90430.log: /var/log/ceph/cluster1-client.mirror.55924.log: /var/log/ceph/cluster1-client.admin.107706.log: /var/log/ceph/cluster1-client.mirror.48302.log: /var/log/ceph/cluster1-client.mirror.101183.log: /var/log/ceph/cluster2-client.admin.62297.log: /var/log/ceph/cluster1-client.mirror.33477.log: /var/log/ceph/cluster2-client.admin.25336.log: /var/log/ceph/cluster1-client.admin.51703.log: /var/log/ceph/cluster1-client.admin.42002.log: /var/log/ceph/cluster2-client.admin.105240.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62297.log.gz 2024-09-18T01:19:04.633 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25336.log.gz 2024-09-18T01:19:04.633 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.86194.log: /var/log/ceph/cluster1-client.mirror.62650.log: /var/log/ceph/cluster1-client.mirror.46053.log: /var/log/ceph/cluster1-client.admin.30389.log: /var/log/ceph/cluster1-client.mirror.81205.log: /var/log/ceph/cluster1-client.admin.49541.log: /var/log/ceph/cluster1-client.admin.117153.log: /var/log/ceph/cluster2-client.admin.73011.log: /var/log/ceph/cluster1-client.admin.82782.log: /var/log/ceph/cluster2-client.mirror.74666.log: /var/log/ceph/cluster1-client.mirror.34740.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73011.log.gz 2024-09-18T01:19:04.633 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.39252.log: /var/log/ceph/cluster1-client.mirror.35159.log: /var/log/ceph/cluster1-client.mirror.90642.log: /var/log/ceph/cluster1-client.admin.130580.log: /var/log/ceph/cluster1-client.mirror.132269.log: /var/log/ceph/cluster1-client.mirror.78630.log: /var/log/ceph/cluster1-client.mirror.82338.log: /var/log/ceph/cluster2-client.admin.25419.log: /var/log/ceph/cluster1-client.mirror.64801.log: /var/log/ceph/cluster1-client.mirror.101877.log: /var/log/ceph/cluster1-client.mirror.48950.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25419.log.gz 2024-09-18T01:19:04.633 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.49399.log: /var/log/ceph/cluster1-client.admin.74506.log: /var/log/ceph/cluster1-client.mirror.58871.log: /var/log/ceph/cluster2-client.admin.95950.log: /var/log/ceph/cluster1-client.mirror.75796.log: /var/log/ceph/cluster1-client.mirror.131427.log: /var/log/ceph/cluster1-client.mirror.56554.log: /var/log/ceph/cluster1-client.mirror.86657.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.95950.log.gz 2024-09-18T01:19:04.633 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87980.log: /var/log/ceph/cluster1-client.mirror.101437.log: /var/log/ceph/cluster1-client.mirror.69706.log: /var/log/ceph/cluster1-client.admin.139079.log: /var/log/ceph/cluster1-client.admin.52345.log: /var/log/ceph/cluster2-client.admin.106012.log: /var/log/ceph/cluster2-client.admin.83969.log: /var/log/ceph/cluster2-client.admin.50377.log: /var/log/ceph/cluster1-client.mirror.136611.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.52345.log.gz 2024-09-18T01:19:04.636 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116589.log: /var/log/ceph/cluster1-client.mirror.68926.log: /var/log/ceph/cluster1-client.mirror.59666.log: /var/log/ceph/cluster1-client.mirror.108115.log: /var/log/ceph/cluster1-client.mirror.132282.log: /var/log/ceph/cluster1-client.mirror.34783.log: /var/log/ceph/cluster1-client.mirror.69033.log: /var/log/ceph/cluster2-client.admin.25487.log: /var/log/ceph/cluster1-client.mirror.137875.log: /var/log/ceph/cluster1-client.admin.30542.log: /var/log/ceph/cluster2-client.mirror.62381.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25487.log.gz 2024-09-18T01:19:04.636 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94518.log: /var/log/ceph/cluster1-client.mirror.113009.log: /var/log/ceph/cluster2-client.mirror.117456.log: /var/log/ceph/cluster1-client.mirror.88792.log: /var/log/ceph/cluster1-client.admin.139358.log: /var/log/ceph/cluster1-client.mirror.31086.log: /var/log/ceph/cluster1-client.admin.107907.log: /var/log/ceph/cluster1-client.mirror.91276.log: /var/log/ceph/cluster1-client.mirror.59624.log: /var/log/ceph/cluster2-client.admin.73566.log: /var/log/ceph/cluster2-client.admin.39387.log: /var/log/ceph/cluster1-client.mirror.79522.log: /var/log/ceph/cluster1-client.mirror.121970.log: /var/log/ceph/cluster2-client.admin.61278.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.39387.log.gz 2024-09-18T01:19:04.636 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.98307.log: /var/log/ceph/cluster1-client.mirror.33480.log: /var/log/ceph/cluster2-client.admin.61705.log: /var/log/ceph/cluster1-client.admin.37822.log: /var/log/ceph/cluster1-client.mirror.111599.log: /var/log/ceph/cluster1-client.mirror.88582.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37822.log.gz 2024-09-18T01:19:04.636 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.62301.log: /var/log/ceph/cluster1-client.mirror.126654.log: /var/log/ceph/cluster1-client.mirror.121912.log: /var/log/ceph/cluster1-client.mirror.91402.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62301.log.gz 2024-09-18T01:19:04.636 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136822.log: /var/log/ceph/cluster1-client.admin.120212.log: /var/log/ceph/cluster1-client.mirror.100260.log: /var/log/ceph/cluster1-client.mirror.111751.log: /var/log/ceph/cluster1-client.mirror.33562.log: /var/log/ceph/cluster1-client.mirror.70334.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.120212.log.gz 2024-09-18T01:19:04.639 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129633.log: /var/log/ceph/cluster1-client.mirror.66399.log: /var/log/ceph/cluster1-client.mirror.69496.log: /var/log/ceph/cluster1-client.mirror.32514.log: /var/log/ceph/cluster1-client.mirror.122747.log: /var/log/ceph/cluster2-client.admin.29346.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129633.log.gz 2024-09-18T01:19:04.639 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.42574.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29346.log.gz 2024-09-18T01:19:04.640 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.78546.log: /var/log/ceph/cluster1-client.mirror.134498.log: /var/log/ceph/cluster2-client.admin.39570.log: /var/log/ceph/cluster1-client.mirror.100934.log: /var/log/ceph/cluster1-client.mirror.76791.log: /var/log/ceph/cluster1-client.admin.31748.log: /var/log/ceph/cluster1-client.mirror.134458.log: /var/log/ceph/cluster1-client.mirror.34214.log: /var/log/ceph/cluster1-client.mirror.43272.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31748.log.gz 2024-09-18T01:19:04.640 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.50989.log: /var/log/ceph/cluster1-client.mirror.80155.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50989.log.gz 2024-09-18T01:19:04.640 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66968.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134499.log.gz 2024-09-18T01:19:04.642 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.71096.log: /var/log/ceph/cluster1-client.admin.61441.log: /var/log/ceph/cluster1-client.mirror.124009.log: /var/log/ceph/cluster1-client.mirror.125519.log: /var/log/ceph/cluster1-client.mirror.32112.log: /var/log/ceph/cluster1-client.admin.118927.log: /var/log/ceph/cluster1-client.mirror.88705.log: /var/log/ceph/cluster2-client.admin.61057.log: /var/log/ceph/cluster1-client.mirror.108642.log: /var/log/ceph/cluster1-client.mirror.34614.log: /var/log/ceph/cluster1-client.mirror.36673.log: /var/log/ceph/cluster1-client.admin.119014.log: /var/log/ceph/cluster1-client.mirror.46302.log: /var/log/ceph/cluster1-client.mirror.88085.log: /var/log/ceph/cluster1-client.mirror.69075.log: /var/log/ceph/cluster2-client.admin.94673.log: /var/log/ceph/cluster1-client.mirror.137873.log: /var/log/ceph/cluster1-client.mirror.35200.log: /var/log/ceph/cluster1-client.mirror.136360.log: /var/log/ceph/cluster1-client.mirror.44724.log: /var/log/ceph/cluster1-client.mirror.102406.log: /var/log/ceph/cluster1-client.mirror.122014.log: /var/log/ceph/cluster1-client.mirror.52158.log: /var/log/ceph/cluster1-client.mirror.114073.log: /var/log/ceph/cluster1-client.admin.40444.log: /var/log/ceph/cluster1-client.mirror.33419.log: /var/log/ceph/cluster1-client.mirror.136820.log: /var/log/ceph/cluster1-client.mirror.34424.log: /var/log/ceph/cluster2-client.admin.140824.log: /var/log/ceph/cluster1-client.admin.53627.log: /var/log/ceph/cluster1-client.mirror.121679.log: /var/log/ceph/cluster2-client.admin.117695.log: /var/log/ceph/cluster1-client.admin.63948.log: /var/log/ceph/cluster1-client.mirror.120596.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140824.log.gz 2024-09-18T01:19:04.642 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.117695.log.gz 2024-09-18T01:19:04.642 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.51011.log: /var/log/ceph/cluster2-client.mirror.129741.log: /var/log/ceph/cluster1-client.mirror.131531.log: /var/log/ceph/cluster1-client.mirror.32422.log: /var/log/ceph/cluster1-client.mirror.55606.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.51011.log.gz 2024-09-18T01:19:04.642 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36738.log: /var/log/ceph/cluster1-client.admin.60328.log: /var/log/ceph/cluster2-client.admin.28995.log: /var/log/ceph/cluster1-client.mirror.80658.log: /var/log/ceph/cluster1-client.admin.127784.log: /var/log/ceph/cluster1-client.mirror.47462.log: /var/log/ceph/cluster1-client.mirror.43740.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28995.log.gz 2024-09-18T01:19:04.656 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74857.log: /var/log/ceph/cluster1-client.admin.64196.log: /var/log/ceph/cluster1-client.mirror.57267.log: /var/log/ceph/cluster2-client.admin.96168.log: /var/log/ceph/cluster1-client.mirror.113491.log: /var/log/ceph/cluster2-client.admin.95393.log: /var/log/ceph/cluster1-client.mirror.113492.log: /var/log/ceph/cluster2-client.mirror.141933.log: /var/log/ceph/cluster1-client.mirror.30505.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96168.log.gz 2024-09-18T01:19:04.656 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.96768.log: /var/log/ceph/cluster1-client.mirror.89592.log: /var/log/ceph/cluster1-client.mirror.71265.log: /var/log/ceph/cluster1-client.mirror.92813.log: /var/log/ceph/cluster2-client.admin.29780.log: /var/log/ceph/cluster1-client.admin.104984.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29780.log.gz 2024-09-18T01:19:04.656 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.141933.log.gz 2024-09-18T01:19:04.656 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.104984.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133195.log: /var/log/ceph/cluster1-client.mirror.89716.log: /var/log/ceph/cluster1-client.mirror.104617.log: /var/log/ceph/cluster1-client.mirror.46997.log: /var/log/ceph/cluster1-client.mirror.132340.log: /var/log/ceph/cluster2-client.admin.107319.log: /var/log/ceph/cluster2-client.admin.118541.log: /var/log/ceph/cluster2-client.admin.61988.log: /var/log/ceph/cluster1-client.mirror.78463.log: /var/log/ceph/cluster1-client.mirror.138561.log: /var/log/ceph/cluster1-client.mirror.45799.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118541.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107319.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87314.log: /var/log/ceph/cluster1-client.mirror.46894.log: /var/log/ceph/cluster1-client.mirror.90079.log: /var/log/ceph/cluster1-client.mirror.56764.log: /var/log/ceph/cluster1-client.mirror.47528.log: /var/log/ceph/cluster1-client.mirror.125310.log: 90.0%/var/log/ceph/cluster1-client.mirror.104007.log: -- replaced with /var/log/ceph/cluster1-client.mirror.115222.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45210.log: /var/log/ceph/cluster2-client.admin.116701.log: /var/log/ceph/cluster2-client.mirror.130197.log: /var/log/ceph/cluster2-client.admin.129261.log: /var/log/ceph/cluster1-client.mirror.30941.log: /var/log/ceph/cluster1-client.mirror.66969.log: /var/log/ceph/cluster1-client.mirror.90263.log: /var/log/ceph/cluster1-client.mirror.101855.log: /var/log/ceph/cluster1-client.mirror.124426.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129261.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.40417.log: /var/log/ceph/cluster2-client.admin.95944.log: /var/log/ceph/cluster2-client.admin.96023.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.95944.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.102319.log: /var/log/ceph/cluster1-client.mirror.98774.log: /var/log/ceph/cluster1-client.mirror.80908.log: /var/log/ceph/cluster1-client.mirror.88180.log: /var/log/ceph/cluster2-client.admin.50059.log: /var/log/ceph/cluster1-client.admin.75149.log: /var/log/ceph/cluster1-client.admin.117973.log: /var/log/ceph/cluster1-client.mirror.127094.log: /var/log/ceph/cluster2-client.admin.62322.log: /var/log/ceph/cluster1-client.mirror.91696.log: /var/log/ceph/cluster1-client.admin.63792.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.75149.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57242.log: /var/log/ceph/cluster1-client.admin.106687.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62322.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.140304.log: /var/log/ceph/cluster1-client.admin.132017.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.63792.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.63576.log: /var/log/ceph/cluster2-client.admin.72495.log: /var/log/ceph/cluster1-client.mirror.88875.log: /var/log/ceph/cluster2-client.mirror.117623.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.132017.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46263.log: /var/log/ceph/cluster1-client.mirror.56552.log: /var/log/ceph/cluster1-client.mirror.80029.log: /var/log/ceph/cluster2-client.admin.49723.log: /var/log/ceph/cluster1-client.admin.96567.log: /var/log/ceph/cluster2-client.admin.61272.log: /var/log/ceph/cluster1-client.mirror.35315.log: /var/log/ceph/cluster1-client.admin.30398.log: /var/log/ceph/cluster1-client.mirror.134714.log: /var/log/ceph/cluster2-client.admin.61679.log: /var/log/ceph/cluster1-client.mirror.44054.log: 0.0%/var/log/ceph/cluster2-client.admin.95678.log: -- replaced with /var/log/ceph/cluster2-client.admin.61272.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65971.log: /var/log/ceph/cluster1-client.mirror.91764.log: /var/log/ceph/cluster1-client.mirror.90851.log: /var/log/ceph/cluster1-client.mirror.52401.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.95678.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133450.log: /var/log/ceph/cluster1-client.mirror.65301.log: /var/log/ceph/cluster2-client.mirror.141839.log: /var/log/ceph/cluster1-client.mirror.133788.log: /var/log/ceph/cluster1-client.admin.98177.log: /var/log/ceph/cluster1-client.mirror.48408.log: /var/log/ceph/cluster1-client.mirror.33060.log: /var/log/ceph/cluster1-client.admin.98010.log: /var/log/ceph/cluster1-client.admin.141106.log: /var/log/ceph/cluster2-client.admin.61101.log: /var/log/ceph/cluster1-client.mirror.69788.log: /var/log/ceph/cluster1-client.mirror.134306.log: /var/log/ceph/cluster1-client.mirror.35518.log: /var/log/ceph/cluster2-client.admin.128832.log: /var/log/ceph/cluster1-client.mirror.103292.log: /var/log/ceph/cluster2-client.admin.62288.log: /var/log/ceph/cluster1-client.mirror.102575.log: /var/log/ceph/cluster1-client.admin.119905.log: /var/log/ceph/cluster1-client.admin.29196.log: /var/log/ceph/cluster1-client.mirror.115433.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62288.log.gz 2024-09-18T01:19:04.657 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29196.log.gz 2024-09-18T01:19:04.658 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91908.log: /var/log/ceph/cluster1-client.mirror.132874.log: /var/log/ceph/cluster1-client.mirror.77204.log: /var/log/ceph/cluster1-client.mirror.64728.log: /var/log/ceph/cluster1-client.mirror.89926.log: /var/log/ceph/cluster1-client.admin.75309.log: /var/log/ceph/cluster1-client.mirror.138658.log: /var/log/ceph/cluster1-client.mirror.47840.log: /var/log/ceph/cluster1-client.mirror.67138.log: /var/log/ceph/cluster1-client.mirror.96406.log: /var/log/ceph/cluster1-client.admin.60833.log: /var/log/ceph/cluster1-client.mirror.112237.log: /var/log/ceph/cluster1-client.mirror.93023.log: /var/log/ceph/cluster2-client.mirror.74190.log: /var/log/ceph/cluster1-client.mirror.120789.log: /var/log/ceph/cluster2-client.admin.50004.log: /var/log/ceph/cluster1-client.admin.84451.log: /var/log/ceph/cluster1-client.admin.105906.log: /var/log/ceph/cluster1-client.mirror.136320.log: /var/log/ceph/cluster1-client.mirror.70082.log: /var/log/ceph/cluster1-client.mirror.105306.log: /var/log/ceph/cluster1-client.mirror.134308.log: /var/log/ceph/cluster1-client.mirror.80867.log: /var/log/ceph/cluster1-client.admin.48859.log: /var/log/ceph/cluster2-client.mirror.84981.log: /var/log/ceph/cluster1-client.mirror.47272.log: /var/log/ceph/cluster1-client.mirror.132842.log: /var/log/ceph/cluster2-client.admin.94764.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48859.log.gz 2024-09-18T01:19:04.658 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.94764.log.gz 2024-09-18T01:19:04.658 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.141106.log.gz 2024-09-18T01:19:04.658 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5%/var/log/ceph/cluster1-client.mirror.91614.log: -- replaced with /var/log/ceph/cluster2-client.admin.95847.log.gz 2024-09-18T01:19:04.658 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.73476.log.gz 2024-09-18T01:19:04.658 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.48158.log: /var/log/ceph/cluster1-client.mirror.115639.log: /var/log/ceph/cluster1-client.mirror.81539.log: /var/log/ceph/cluster1-client.mirror.68018.log: /var/log/ceph/cluster1-client.mirror.82253.log: /var/log/ceph/cluster1-client.admin.86238.log: /var/log/ceph/cluster1-client.admin.76460.log: /var/log/ceph/cluster1-client.mirror.99498.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.95845.log.gz 2024-09-18T01:19:04.658 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.30819.log: /var/log/ceph/cluster1-client.admin.51019.log: /var/log/ceph/cluster1-client.mirror.114000.log: /var/log/ceph/cluster1-client.mirror.92242.log: /var/log/ceph/cluster1-client.mirror.77916.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30819.log.gz 2024-09-18T01:19:04.658 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.130124.log: /var/log/ceph/cluster1-client.mirror.55224.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.95369.log.gz 2024-09-18T01:19:04.660 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74994.log: /var/log/ceph/cluster1-client.mirror.56217.log: /var/log/ceph/cluster2-client.admin.129281.log: /var/log/ceph/cluster1-client.mirror.114128.log: /var/log/ceph/cluster1-client.mirror.77789.log: /var/log/ceph/cluster1-client.admin.140878.log: /var/log/ceph/cluster2-client.admin.117696.log: /var/log/ceph/cluster2-client.mirror.96499.log: /var/log/ceph/cluster1-client.mirror.66234.log: /var/log/ceph/cluster1-client.admin.139893.log: /var/log/ceph/cluster1-client.admin.94458.log: /var/log/ceph/cluster1-client.admin.73909.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.117696.log.gz 2024-09-18T01:19:04.660 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.112440.log: /var/log/ceph/cluster1-client.mirror.124845.log: /var/log/ceph/cluster1-client.mirror.77498.log: /var/log/ceph/cluster2-client.mirror.60974.log: /var/log/ceph/cluster1-client.mirror.100638.log: /var/log/ceph/cluster1-client.mirror.33375.log: /var/log/ceph/cluster1-client.admin.71805.log: /var/log/ceph/cluster1-client.mirror.37973.log: /var/log/ceph/cluster1-client.mirror.135941.log: /var/log/ceph/cluster1-client.mirror.66967.log: /var/log/ceph/cluster1-client.mirror.90053.log: /var/log/ceph/cluster1-client.mirror.76729.log: /var/log/ceph/cluster1-client.mirror.112273.log: /var/log/ceph/cluster1-client.mirror.103333.log: /var/log/ceph/cluster2-client.mirror.74552.log: /var/log/ceph/cluster1-client.admin.30976.log: /var/log/ceph/cluster1-client.mirror.112168.log: /var/log/ceph/cluster1-client.admin.107592.log: /var/log/ceph/cluster2-client.admin.140838.log: /var/log/ceph/cluster1-client.admin.140303.log: /var/log/ceph/cluster1-client.mirror.108754.log: /var/log/ceph/cluster1-client.mirror.63106.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140838.log.gz 2024-09-18T01:19:04.660 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31180.log: /var/log/ceph/cluster1-client.mirror.67261.log: /var/log/ceph/cluster1-client.mirror.134710.log: /var/log/ceph/cluster1-client.admin.141993.log: /var/log/ceph/cluster1-client.mirror.103629.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.141993.log.gz 2024-09-18T01:19:04.660 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.54908.log: /var/log/ceph/cluster2-client.admin.39274.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.85049.log.gz 2024-09-18T01:19:04.661 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.140606.log: /var/log/ceph/cluster1-client.mirror.124259.log: /var/log/ceph/cluster1-client.admin.40987.log: /var/log/ceph/cluster2-client.admin.40039.log: /var/log/ceph/cluster1-client.mirror.44832.log: /var/log/ceph/cluster1-client.mirror.137200.log: /var/log/ceph/cluster1-client.mirror.130218.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.40039.log.gz 2024-09-18T01:19:04.661 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.28665.log: /var/log/ceph/cluster1-client.mirror.122722.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28665.log.gz 2024-09-18T01:19:04.662 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.46472.log: /var/log/ceph/cluster1-client.admin.119584.log: /var/log/ceph/cluster1-client.mirror.58616.log: /var/log/ceph/cluster1-client.mirror.98105.log: /var/log/ceph/cluster1-client.mirror.131913.log: /var/log/ceph/cluster2-client.mirror.40926.log: /var/log/ceph/cluster1-client.admin.73310.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.140878.log.gz 2024-09-18T01:19:04.662 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.41154.log: /var/log/ceph/cluster1-client.mirror.77370.log: /var/log/ceph/cluster2-client.admin.50998.log: 19.9% -- replaced with /var/log/ceph/cluster2-client.admin.117706.log.gz 2024-09-18T01:19:04.807 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50998.log.gz 2024-09-18T01:19:04.807 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2%/var/log/ceph/cluster1-client.mirror.30204.log: -- replaced with /var/log/ceph/cluster2-client.mirror.141839.log.gz 2024-09-18T01:19:04.807 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56218.log: /var/log/ceph/cluster1-client.mirror.67516.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.117922.log.gz 2024-09-18T01:19:04.807 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93992.log: /var/log/ceph/cluster1-client.mirror.32640.log: /var/log/ceph/cluster1-client.mirror.37157.log: /var/log/ceph/cluster1-client.admin.52633.log: /var/log/ceph/cluster1-client.mirror.66634.log: /var/log/ceph/cluster1-client.mirror.100304.log: /var/log/ceph/cluster1-client.mirror.125730.log: /var/log/ceph/cluster1-client.mirror.137832.log: /var/log/ceph/cluster1-client.mirror.88748.log: /var/log/ceph/cluster1-client.mirror.41876.log: /var/log/ceph/cluster2-client.admin.128645.log: /var/log/ceph/cluster1-client.mirror.135981.log: /var/log/ceph/cluster1-client.mirror.81918.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.63633.log.gz 2024-09-18T01:19:04.808 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138666.log: /var/log/ceph/cluster1-client.mirror.81665.log: /var/log/ceph/cluster1-client.mirror.133406.log: /var/log/ceph/cluster1-client.admin.98527.log: /var/log/ceph/cluster1-client.admin.61468.log: /var/log/ceph/cluster1-client.admin.64350.log: /var/log/ceph/cluster1-client.mirror.33794.log: /var/log/ceph/cluster1-client.admin.98798.log: /var/log/ceph/cluster1-client.mirror.114338.log: /var/log/ceph/cluster1-client.mirror.57394.log: /var/log/ceph/cluster2-client.admin.140681.log: /var/log/ceph/cluster1-client.mirror.99925.log: /var/log/ceph/cluster1-client.mirror.48261.log: 0.0%/var/log/ceph/cluster1-client.mirror.69999.log: -- replaced with /var/log/ceph/cluster1-client.admin.64350.log.gz 2024-09-18T01:19:04.808 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.74210.log: /var/log/ceph/cluster1-client.mirror.135130.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140681.log.gz 2024-09-18T01:19:04.808 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.44200.log: /var/log/ceph/cluster1-client.admin.121360.log: /var/log/ceph/cluster1-client.admin.40303.log: /var/log/ceph/cluster1-client.mirror.42048.log: /var/log/ceph/cluster1-client.mirror.91841.log: /var/log/ceph/cluster2-client.mirror.73868.log: /var/log/ceph/cluster1-client.mirror.34319.log: /var/log/ceph/cluster1-client.mirror.78755.log: /var/log/ceph/cluster1-client.mirror.100385.log: /var/log/ceph/cluster2-client.admin.60762.log: /var/log/ceph/cluster1-client.mirror.46850.log: /var/log/ceph/cluster1-client.admin.85478.log: /var/log/ceph/cluster1-client.mirror.57602.log: /var/log/ceph/cluster1-client.mirror.68652.log: /var/log/ceph/cluster1-client.admin.96909.log: /var/log/ceph/cluster1-client.mirror.114590.log: /var/log/ceph/cluster1-client.admin.140215.log: /var/log/ceph/cluster1-client.mirror.55014.log: /var/log/ceph/cluster1-client.mirror.57772.log: /var/log/ceph/cluster1-client.admin.71806.log: /var/log/ceph/cluster1-client.mirror.78883.log: /var/log/ceph/cluster2-client.admin.129354.log: /var/log/ceph/cluster1-client.mirror.70629.log: /var/log/ceph/cluster1-client.mirror.138004.log: /var/log/ceph/cluster1-client.mirror.98842.log: /var/log/ceph/cluster1-client.admin.60096.log: /var/log/ceph/cluster1-client.mirror.127116.log: /var/log/ceph/cluster1-client.admin.107934.log: /var/log/ceph/cluster1-client.mirror.75524.log: /var/log/ceph/cluster1-client.mirror.31355.log: /var/log/ceph/cluster1-client.mirror.125520.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60096.log.gz 2024-09-18T01:19:04.808 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56467.log: /var/log/ceph/cluster1-client.admin.97137.log: /var/log/ceph/cluster1-client.mirror.98484.log: /var/log/ceph/cluster1-client.mirror.54807.log: /var/log/ceph/cluster1-client.mirror.58154.log: /var/log/ceph/cluster2-client.admin.50111.log: /var/log/ceph/cluster1-client.mirror.78839.log: /var/log/ceph/cluster1-client.admin.72128.log: /var/log/ceph/cluster1-client.mirror.125186.log: /var/log/ceph/cluster1-client.mirror.98049.log: /var/log/ceph/cluster2-client.admin.140812.log: /var/log/ceph/cluster1-client.admin.108545.log: /var/log/ceph/cluster1-client.mirror.134141.log: /var/log/ceph/cluster1-client.mirror.85572.log: /var/log/ceph/cluster2-client.admin.39689.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140812.log.gz 2024-09-18T01:19:04.808 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.57945.log: /var/log/ceph/cluster1-client.mirror.31179.log: /var/log/ceph/cluster1-client.mirror.113744.log: /var/log/ceph/cluster1-client.mirror.124761.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.39689.log.gz 2024-09-18T01:19:04.808 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.140828.log: /var/log/ceph/cluster1-client.mirror.113788.log: /var/log/ceph/cluster1-client.mirror.123629.log: /var/log/ceph/cluster1-client.mirror.55434.log: /var/log/ceph/cluster1-client.mirror.36253.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140828.log.gz 2024-09-18T01:19:04.808 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.77499.log: /var/log/ceph/cluster2-client.mirror.39962.log: /var/log/ceph/cluster1-client.admin.31550.log: /var/log/ceph/cluster1-client.mirror.34737.log: /var/log/ceph/cluster2-client.mirror.40356.log: /var/log/ceph/cluster1-client.mirror.33627.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31550.log.gz 2024-09-18T01:19:04.808 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.33313.log: /var/log/ceph/cluster1-client.mirror.123000.log: /var/log/ceph/cluster1-client.admin.63665.log: /var/log/ceph/cluster1-client.mirror.100764.log: /var/log/ceph/cluster1-client.mirror.67390.log: /var/log/ceph/cluster1-client.mirror.102574.log: /var/log/ceph/cluster1-client.admin.132019.log: /var/log/ceph/cluster1-client.mirror.45774.log: /var/log/ceph/cluster1-client.mirror.93509.log: /var/log/ceph/cluster1-client.mirror.114337.log: /var/log/ceph/cluster1-client.mirror.82126.log: /var/log/ceph/cluster1-client.mirror.58030.log: /var/log/ceph/cluster1-client.mirror.98154.log: /var/log/ceph/cluster1-client.mirror.111851.log: /var/log/ceph/cluster1-client.mirror.55328.log: /var/log/ceph/cluster2-client.admin.50980.log: /var/log/ceph/cluster2-client.admin.50320.log: /var/log/ceph/cluster1-client.mirror.35727.log: /var/log/ceph/cluster1-client.mirror.47062.log: /var/log/ceph/cluster1-client.mirror.103838.log: /var/log/ceph/cluster1-client.mirror.92393.log: /var/log/ceph/cluster1-client.mirror.55982.log: /var/log/ceph/cluster1-client.mirror.45000.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50980.log.gz 2024-09-18T01:19:04.808 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45251.log: /var/log/ceph/cluster1-client.mirror.115640.log: /var/log/ceph/cluster1-client.mirror.108343.log: /var/log/ceph/cluster1-client.admin.41704.log: /var/log/ceph/cluster1-client.mirror.56194.log: /var/log/ceph/cluster1-client.mirror.103462.log: /var/log/ceph/cluster2-client.admin.117486.log: /var/log/ceph/cluster1-client.mirror.103461.log: /var/log/ceph/cluster1-client.mirror.32316.log: /var/log/ceph/cluster1-client.mirror.43574.log: /var/log/ceph/cluster1-client.mirror.138401.log: /var/log/ceph/cluster1-client.mirror.131621.log: /var/log/ceph/cluster1-client.mirror.35475.log: /var/log/ceph/cluster1-client.mirror.86776.log: /var/log/ceph/cluster1-client.mirror.88915.log: /var/log/ceph/cluster1-client.mirror.69766.log: /var/log/ceph/cluster1-client.mirror.63771.log: /var/log/ceph/cluster1-client.mirror.42113.log: /var/log/ceph/cluster1-client.mirror.70929.log: /var/log/ceph/cluster1-client.mirror.36949.log: /var/log/ceph/cluster1-client.mirror.78000.log: /var/log/ceph/cluster1-client.mirror.70885.log: /var/log/ceph/cluster1-client.mirror.121845.log: /var/log/ceph/cluster2-client.admin.61041.log: /var/log/ceph/cluster1-client.mirror.137664.log: /var/log/ceph/cluster1-client.mirror.118880.log: /var/log/ceph/cluster2-client.mirror.29925.log: /var/log/ceph/cluster2-client.admin.73126.log: /var/log/ceph/cluster1-client.mirror.115758.log: /var/log/ceph/cluster2-client.admin.139574.log: /var/log/ceph/cluster2-client.admin.140806.log: /var/log/ceph/cluster1-client.mirror.112588.log: /var/log/ceph/cluster1-client.mirror.100682.log: /var/log/ceph/cluster1-client.admin.41274.log: /var/log/ceph/cluster1-client.mirror.100553.log: /var/log/ceph/cluster1-client.mirror.87598.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140806.log.gz 2024-09-18T01:19:04.808 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133667.log: /var/log/ceph/cluster1-client.mirror.43632.log: /var/log/ceph/cluster1-client.mirror.56007.log: /var/log/ceph/cluster2-client.admin.60712.log: /var/log/ceph/cluster1-client.mirror.100806.log: /var/log/ceph/cluster1-client.mirror.59983.log: /var/log/ceph/cluster1-client.mirror.58532.log: /var/log/ceph/cluster1-client.mirror.45638.log: /var/log/ceph/cluster1-client.mirror.123165.log: /var/log/ceph/cluster1-client.mirror.79301.log: /var/log/ceph/cluster1-client.mirror.100346.log: /var/log/ceph/cluster2-client.admin.96156.log: /var/log/ceph/cluster1-client.mirror.136191.log: /var/log/ceph/cluster1-client.mirror.111162.log: /var/log/ceph/cluster1-client.mirror.111390.log: /var/log/ceph/cluster1-client.mirror.134500.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96156.log.gz 2024-09-18T01:19:04.808 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138082.log: /var/log/ceph/cluster2-client.admin.84884.log: /var/log/ceph/cluster1-client.mirror.29652.log: /var/log/ceph/cluster1-client.mirror.48470.log: /var/log/ceph/cluster1-client.mirror.134417.log: /var/log/ceph/cluster2-client.admin.50128.log: /var/log/ceph/cluster1-client.admin.49587.log: /var/log/ceph/cluster1-client.mirror.55118.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84884.log.gz 2024-09-18T01:19:04.809 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51300.log: /var/log/ceph/cluster1-client.mirror.43674.log: /var/log/ceph/cluster2-client.mirror.72808.log: /var/log/ceph/cluster1-client.admin.138621.log: /var/log/ceph/cluster1-client.mirror.46196.log: /var/log/ceph/cluster1-client.mirror.132264.log: /var/log/ceph/cluster1-client.mirror.43741.log: /var/log/ceph/cluster1-client.mirror.36213.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.138621.log.gz 2024-09-18T01:19:04.809 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.34822.log: /var/log/ceph/cluster2-client.mirror.63407.log: /var/log/ceph/cluster1-client.mirror.86084.log: /var/log/ceph/cluster1-client.mirror.54250.log: /var/log/ceph/cluster1-client.mirror.114421.log: /var/log/ceph/cluster1-client.mirror.130674.log: /var/log/ceph/cluster1-client.mirror.93379.log: /var/log/ceph/cluster1-client.mirror.43129.log: /var/log/ceph/cluster1-client.mirror.78335.log: /var/log/ceph/cluster1-client.mirror.131883.log: /var/log/ceph/cluster1-client.mirror.126085.log: /var/log/ceph/cluster1-client.mirror.36715.log: /var/log/ceph/cluster1-client.mirror.102619.log: /var/log/ceph/cluster1-client.mirror.136613.log: /var/log/ceph/cluster1-client.mirror.71055.log: /var/log/ceph/cluster2-client.mirror.96593.log: /var/log/ceph/cluster1-client.mirror.136321.log: /var/log/ceph/cluster2-client.admin.72078.log: /var/log/ceph/cluster1-client.mirror.100134.log: /var/log/ceph/cluster1-client.mirror.92454.log: /var/log/ceph/cluster1-client.mirror.46365.log: /var/log/ceph/cluster1-client.mirror.66377.log: /var/log/ceph/cluster1-client.admin.74251.log: /var/log/ceph/cluster1-client.admin.62583.log: /var/log/ceph/cluster1-client.admin.121134.log: /var/log/ceph/cluster1-client.mirror.78464.log: /var/log/ceph/cluster1-client.mirror.123564.log: /var/log/ceph/cluster1-client.mirror.34005.log: /var/log/ceph/cluster1-client.mirror.36359.log: /var/log/ceph/cluster1-client.mirror.130916.log: /var/log/ceph/cluster1-client.mirror.81581.log: /var/log/ceph/cluster1-client.mirror.69412.log: /var/log/ceph/cluster2-client.mirror.95570.log: /var/log/ceph/cluster1-client.mirror.112948.log: /var/log/ceph/cluster2-client.admin.39706.log: /var/log/ceph/cluster2-client.mirror.0.30015.log: /var/log/ceph/cluster1-client.mirror.65885.log: /var/log/ceph/cluster1-client.mirror.120661.log: /var/log/ceph/cluster1-client.mirror.136951.log: /var/log/ceph/cluster1-client.admin.83543.log: /var/log/ceph/cluster1-client.admin.97845.log: /var/log/ceph/cluster1-client.admin.30387.log: /var/log/ceph/cluster2-client.mirror.41174.log: /var/log/ceph/cluster2-client.admin.83661.log: /var/log/ceph/cluster1-client.mirror.89212.log: /var/log/ceph/cluster1-client.mirror.127242.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30387.log.gz 2024-09-18T01:19:04.809 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.97845.log.gz 2024-09-18T01:19:04.809 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.29146.log: /var/log/ceph/cluster1-client.mirror.134078.log: /var/log/ceph/cluster1-client.mirror.123840.log: /var/log/ceph/cluster1-client.admin.128322.log: /var/log/ceph/cluster1-client.admin.61550.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29146.log.gz 2024-09-18T01:19:04.809 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31601.log: /var/log/ceph/cluster1-client.mirror.59349.log: /var/log/ceph/cluster2-client.admin.73449.log: /var/log/ceph/cluster1-client.mirror.75698.log: /var/log/ceph/cluster1-client.admin.108955.log: /var/log/ceph/cluster2-client.mirror.95563.log: /var/log/ceph/cluster1-client.mirror.57184.log: /var/log/ceph/cluster1-client.mirror.110586.log: /var/log/ceph/cluster1-client.mirror.113787.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.108955.log.gz 2024-09-18T01:19:04.809 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89927.log: /var/log/ceph/cluster1-client.admin.106730.log: /var/log/ceph/cluster1-client.mirror.132725.log: /var/log/ceph/cluster1-client.mirror.43781.log: /var/log/ceph/cluster1-client.admin.50490.log: /var/log/ceph/cluster1-client.admin.61542.log: /var/log/ceph/cluster2-client.admin.95237.log: /var/log/ceph/cluster1-client.mirror.112859.log: /var/log/ceph/cluster2-client.admin.139759.log: /var/log/ceph/cluster1-client.mirror.90705.log: /var/log/ceph/cluster1-client.mirror.32893.log: /var/log/ceph/cluster1-client.admin.75659.log: /var/log/ceph/cluster1-client.mirror.131085.log: /var/log/ceph/cluster1-client.mirror.70816.log: /var/log/ceph/cluster1-client.mirror.70294.log: /var/log/ceph/cluster1-client.mirror.35410.log: /var/log/ceph/cluster2-client.mirror.130633.log: /var/log/ceph/cluster1-client.mirror.122803.log: /var/log/ceph/cluster1-client.mirror.112523.log: /var/log/ceph/cluster1-client.admin.50642.log: /var/log/ceph/cluster1-client.admin.52495.log: /var/log/ceph/cluster1-client.mirror.44538.log: /var/log/ceph/cluster1-client.mirror.67767.log: /var/log/ceph/cluster1-client.mirror.43908.log: /var/log/ceph/cluster1-client.mirror.122958.log: /var/log/ceph/cluster1-client.admin.106756.log: /var/log/ceph/cluster1-client.admin.119757.log: /var/log/ceph/cluster1-client.mirror.58954.log: /var/log/ceph/cluster1-client.admin.138887.log: /var/log/ceph/cluster2-client.admin.116869.log: /var/log/ceph/cluster1-client.mirror.75240.log: /var/log/ceph/cluster1-client.mirror.78757.log: /var/log/ceph/cluster1-client.mirror.36505.log: /var/log/ceph/cluster1-client.mirror.67597.log: /var/log/ceph/cluster1-client.mirror.44160.log: /var/log/ceph/cluster1-client.mirror.30618.log: /var/log/ceph/cluster2-client.admin.38928.log: /var/log/ceph/cluster1-client.mirror.87562.log: /var/log/ceph/cluster1-client.mirror.55270.log: /var/log/ceph/cluster1-client.mirror.67262.log: /var/log/ceph/cluster1-client.mirror.46220.log: /var/log/ceph/cluster1-client.mirror.126925.log: /var/log/ceph/cluster1-client.admin.60225.log: /var/log/ceph/cluster1-client.mirror.131596.log: /var/log/ceph/cluster1-client.admin.119755.log: /var/log/ceph/cluster1-client.mirror.122955.log: /var/log/ceph/cluster1-client.mirror.99091.log: /var/log/ceph/cluster1-client.mirror.88179.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60225.log.gz 2024-09-18T01:19:04.809 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.51797.log: /var/log/ceph/cluster2-client.mirror.130653.log: /var/log/ceph/cluster1-client.mirror.109470.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.119755.log.gz 2024-09-18T01:19:04.809 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109316.log: /var/log/ceph/cluster1-client.mirror.133787.log: /var/log/ceph/cluster1-client.mirror.110375.log: /var/log/ceph/cluster1-client.admin.87093.log: /var/log/ceph/cluster1-client.mirror.57348.log: /var/log/ceph/cluster2-client.mirror.95648.log: /var/log/ceph/cluster1-client.admin.127617.log: /var/log/ceph/cluster2-client.admin.105735.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.87093.log.gz 2024-09-18T01:19:04.809 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31857.log: /var/log/ceph/cluster1-client.mirror.111539.log: /var/log/ceph/cluster2-client.admin.84561.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.127617.log.gz 2024-09-18T01:19:04.809 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.100345.log: /var/log/ceph/cluster1-client.mirror.124215.log: /var/log/ceph/cluster1-client.mirror.91195.log: /var/log/ceph/cluster2-client.admin.118391.log: /var/log/ceph/cluster1-client.mirror.124637.log: /var/log/ceph/cluster1-client.mirror.48471.log: /var/log/ceph/cluster1-client.mirror.92370.log: /var/log/ceph/cluster1-client.mirror.52601.log: /var/log/ceph/cluster1-client.mirror.32849.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.118391.log.gz 2024-09-18T01:19:04.809 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.55879.log: /var/log/ceph/cluster2-client.admin.73037.log: /var/log/ceph/cluster2-client.admin.128712.log: /var/log/ceph/cluster1-client.mirror.109551.log: /var/log/ceph/cluster1-client.mirror.131047.log: /var/log/ceph/cluster1-client.admin.71840.log: /var/log/ceph/cluster1-client.mirror.125731.log: /var/log/ceph/cluster1-client.mirror.68291.log: /var/log/ceph/cluster1-client.admin.48912.log: /var/log/ceph/cluster1-client.admin.127994.log: /var/log/ceph/cluster1-client.mirror.108913.log: /var/log/ceph/cluster1-client.mirror.75760.log: /var/log/ceph/cluster1-client.mirror.41385.log: /var/log/ceph/cluster1-client.mirror.88246.log: /var/log/ceph/cluster1-client.mirror.63706.log: /var/log/ceph/cluster1-client.admin.118558.log: /var/log/ceph/cluster1-client.mirror.131556.log: /var/log/ceph/cluster1-client.admin.119128.log: /var/log/ceph/cluster1-client.mirror.78673.log: /var/log/ceph/cluster1-client.mirror.70754.log: /var/log/ceph/cluster2-client.admin.117694.log: /var/log/ceph/cluster2-client.mirror.96727.log: /var/log/ceph/cluster1-client.admin.60834.log: /var/log/ceph/cluster1-client.mirror.118652.log: /var/log/ceph/cluster2-client.admin.39737.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.117694.log.gz 2024-09-18T01:19:04.810 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.60104.log: /var/log/ceph/cluster1-client.mirror.59559.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60104.log.gz 2024-09-18T01:19:04.810 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.103544.log: /var/log/ceph/cluster1-client.mirror.114064.log: /var/log/ceph/cluster1-client.mirror.123354.log: /var/log/ceph/cluster1-client.mirror.65046.log: /var/log/ceph/cluster1-client.admin.82731.log: /var/log/ceph/cluster1-client.mirror.35728.log: /var/log/ceph/cluster1-client.admin.53175.log: /var/log/ceph/cluster1-client.mirror.35685.log: /var/log/ceph/cluster1-client.mirror.93000.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.82731.log.gz 2024-09-18T01:19:04.810 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.37963.log: /var/log/ceph/cluster1-client.mirror.52463.log: /var/log/ceph/cluster2-client.admin.95685.log: /var/log/ceph/cluster1-client.mirror.103754.log: /var/log/ceph/cluster2-client.admin.106517.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37963.log.gz 2024-09-18T01:19:04.810 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110141.log: /var/log/ceph/cluster1-client.mirror.68185.log: /var/log/ceph/cluster1-client.mirror.132779.log: /var/log/ceph/cluster1-client.mirror.90346.log: /var/log/ceph/cluster1-client.mirror.46684.log: /var/log/ceph/cluster1-client.mirror.134841.log: /var/log/ceph/cluster2-client.admin.139539.log: /var/log/ceph/cluster1-client.admin.117138.log: /var/log/ceph/cluster1-client.mirror.47571.log: /var/log/ceph/cluster1-client.mirror.114801.log: /var/log/ceph/cluster1-client.admin.109111.log: /var/log/ceph/cluster1-client.mirror.36842.log: /var/log/ceph/cluster1-client.mirror.48053.log: /var/log/ceph/cluster2-client.mirror.119657.log: /var/log/ceph/cluster1-client.mirror.93083.log: /var/log/ceph/cluster1-client.mirror.47780.log: /var/log/ceph/cluster1-client.mirror.30715.log: /var/log/ceph/cluster1-client.admin.141535.log: /var/log/ceph/cluster1-client.mirror.69789.log: /var/log/ceph/cluster2-client.mirror.85873.log: /var/log/ceph/cluster1-client.mirror.54674.log: /var/log/ceph/cluster2-client.mirror.29959.log: /var/log/ceph/cluster1-client.mirror.46724.log: /var/log/ceph/cluster1-client.mirror.124342.log: /var/log/ceph/cluster1-client.mirror.37995.log: /var/log/ceph/cluster2-client.mirror.119543.log: /var/log/ceph/cluster2-client.mirror.119087.log: /var/log/ceph/cluster1-client.mirror.55269.log: /var/log/ceph/cluster1-client.mirror.120525.log: /var/log/ceph/cluster1-client.mirror.77047.log: /var/log/ceph/cluster1-client.mirror.64934.log: /var/log/ceph/cluster1-client.mirror.66274.log: /var/log/ceph/cluster1-client.mirror.65625.log: /var/log/ceph/cluster1-client.mirror.123588.log: /var/log/ceph/cluster1-client.mirror.34676.log: /var/log/ceph/cluster1-client.mirror.123795.log: /var/log/ceph/cluster1-client.mirror.41741.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.141535.log.gz 2024-09-18T01:19:04.810 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.76469.log: /var/log/ceph/cluster1-client.mirror.109003.log: /var/log/ceph/cluster1-client.mirror.48118.log: /var/log/ceph/cluster1-client.mirror.0.29986.log: /var/log/ceph/cluster1-client.admin.117021.log: /var/log/ceph/cluster1-client.mirror.97894.log: /var/log/ceph/cluster2-client.admin.140810.log: /var/log/ceph/cluster1-client.mirror.77497.log: /var/log/ceph/cluster1-client.admin.106378.log: /var/log/ceph/cluster1-client.mirror.47421.log: /var/log/ceph/cluster1-client.admin.86107.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140810.log.gz 2024-09-18T01:19:04.810 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69160.log: /var/log/ceph/cluster1-client.mirror.58994.log: /var/log/ceph/cluster1-client.mirror.70712.log: /var/log/ceph/cluster1-client.mirror.99119.log: /var/log/ceph/cluster1-client.mirror.58299.log: /var/log/ceph/cluster1-client.admin.140965.log: /var/log/ceph/cluster2-client.mirror.71989.log: /var/log/ceph/cluster1-client.mirror.98085.log: /var/log/ceph/cluster1-client.mirror.32850.log: /var/log/ceph/cluster1-client.mirror.104092.log: /var/log/ceph/cluster1-client.admin.98536.log: /var/log/ceph/cluster1-client.mirror.68524.log: /var/log/ceph/cluster1-client.mirror.44537.log: /var/log/ceph/cluster1-client.mirror.126527.log: /var/log/ceph/cluster1-client.mirror.64299.log: /var/log/ceph/cluster2-client.mirror.41060.log: /var/log/ceph/cluster2-client.admin.129472.log: /var/log/ceph/cluster1-client.mirror.40168.log: /var/log/ceph/cluster1-client.admin.94945.log: /var/log/ceph/cluster1-client.mirror.111327.log: /var/log/ceph/cluster1-client.admin.138497.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.140965.log.gz 2024-09-18T01:19:04.810 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99159.log: /var/log/ceph/cluster1-client.mirror.140945.log: /var/log/ceph/cluster1-client.mirror.114700.log: /var/log/ceph/cluster1-client.mirror.78167.log: /var/log/ceph/cluster1-client.mirror.46306.log: /var/log/ceph/cluster1-client.admin.86582.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.138497.log.gz 2024-09-18T01:19:04.810 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.56088.log: 66.6%/var/log/ceph/cluster1-client.mirror.109675.log: -- replaced with /var/log/ceph/cluster1-client.mirror.140945.log.gz 2024-09-18T01:19:04.810 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67935.log: /var/log/ceph/cluster2-client.mirror.61656.log: /var/log/ceph/cluster1-client.mirror.138318.log: /var/log/ceph/cluster1-client.mirror.67601.log: /var/log/ceph/cluster1-client.mirror.44895.log: /var/log/ceph/cluster1-client.mirror.43884.log: /var/log/ceph/cluster1-client.admin.29071.log: /var/log/ceph/cluster1-client.mirror.31603.log: /var/log/ceph/cluster1-client.admin.128314.log: /var/log/ceph/cluster1-client.mirror.54030.log: /var/log/ceph/cluster1-client.mirror.40625.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29071.log.gz 2024-09-18T01:19:04.810 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133237.log: /var/log/ceph/cluster2-client.mirror.84760.log: /var/log/ceph/cluster1-client.mirror.91485.log: /var/log/ceph/cluster1-client.mirror.92873.log: /var/log/ceph/cluster1-client.admin.64352.log: /var/log/ceph/cluster1-client.mirror.89656.log: /var/log/ceph/cluster2-client.admin.142065.log: /var/log/ceph/cluster2-client.admin.128926.log: /var/log/ceph/cluster1-client.mirror.123210.log: /var/log/ceph/cluster1-client.admin.82602.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142065.log.gz 2024-09-18T01:19:04.810 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.82602.log.gz 2024-09-18T01:19:04.810 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.58533.log: /var/log/ceph/cluster1-client.mirror.99539.log: /var/log/ceph/cluster1-client.mirror.92456.log: /var/log/ceph/cluster1-client.mirror.45883.log: /var/log/ceph/cluster1-client.mirror.124343.log: /var/log/ceph/cluster2-client.mirror.119409.log: /var/log/ceph/cluster2-client.admin.39752.log: /var/log/ceph/cluster1-client.mirror.125265.log: /var/log/ceph/cluster1-client.mirror.63504.log: /var/log/ceph/cluster1-client.admin.138830.log: /var/log/ceph/cluster2-client.admin.73552.log: /var/log/ceph/cluster1-client.mirror.120115.log: /var/log/ceph/cluster1-client.mirror.136740.log: /var/log/ceph/cluster1-client.admin.106109.log: /var/log/ceph/cluster1-client.mirror.123501.log: /var/log/ceph/cluster1-client.mirror.101645.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73552.log.gz 2024-09-18T01:19:04.811 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80448.log: /var/log/ceph/cluster1-client.admin.129162.log: /var/log/ceph/cluster2-client.admin.129643.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129643.log.gz 2024-09-18T01:19:04.811 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104553.log: /var/log/ceph/cluster1-client.admin.94485.log: /var/log/ceph/cluster1-client.mirror.32892.log: /var/log/ceph/cluster1-client.mirror.125606.log: /var/log/ceph/cluster1-client.mirror.41933.log: /var/log/ceph/cluster1-client.admin.96339.log: /var/log/ceph/cluster1-client.mirror.67809.log: /var/log/ceph/cluster1-client.mirror.105137.log: /var/log/ceph/cluster1-client.mirror.75997.log: /var/log/ceph/cluster1-client.mirror.87876.log: /var/log/ceph/cluster1-client.admin.108957.log: /var/log/ceph/cluster1-client.admin.86116.log: /var/log/ceph/cluster1-client.mirror.48260.log: /var/log/ceph/cluster1-client.admin.74479.log: /var/log/ceph/cluster1-client.admin.117134.log: /var/log/ceph/cluster1-client.mirror.52538.log: /var/log/ceph/cluster1-client.mirror.119714.log: /var/log/ceph/cluster2-client.admin.73038.log: /var/log/ceph/cluster1-client.mirror.98573.log: /var/log/ceph/cluster2-client.admin.94080.log: /var/log/ceph/cluster2-client.mirror.51414.log: /var/log/ceph/cluster1-client.mirror.78209.log: /var/log/ceph/cluster1-client.admin.131143.log: /var/log/ceph/cluster1-client.mirror.33899.log: /var/log/ceph/cluster1-client.mirror.138127.log: /var/log/ceph/cluster1-client.mirror.54849.log: /var/log/ceph/cluster1-client.admin.38397.log: /var/log/ceph/cluster1-client.mirror.122324.log: /var/log/ceph/cluster1-client.mirror.30167.log: /var/log/ceph/cluster2-client.admin.140842.log: /var/log/ceph/cluster1-client.mirror.87831.log: /var/log/ceph/cluster2-client.admin.83607.log: /var/log/ceph/cluster1-client.mirror.135050.log: /var/log/ceph/cluster1-client.admin.71694.log: /var/log/ceph/cluster1-client.mirror.67556.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140842.log.gz 2024-09-18T01:19:04.811 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.88286.log: /var/log/ceph/cluster1-client.admin.93815.log: /var/log/ceph/cluster1-client.admin.116206.log: /var/log/ceph/cluster1-client.mirror.132170.log: /var/log/ceph/cluster1-client.mirror.92536.log: /var/log/ceph/cluster1-client.mirror.30777.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.93815.log.gz 2024-09-18T01:19:04.811 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136531.log: /var/log/ceph/cluster1-client.admin.50420.log: /var/log/ceph/cluster2-client.admin.62043.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.116206.log.gz 2024-09-18T01:19:04.811 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59625.log: /var/log/ceph/cluster1-client.mirror.58660.log: /var/log/ceph/cluster1-client.admin.37834.log: /var/log/ceph/cluster1-client.mirror.63907.log: /var/log/ceph/cluster1-client.mirror.70907.log: /var/log/ceph/cluster2-client.mirror.108436.log: /var/log/ceph/cluster1-client.mirror.48515.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37834.log.gz 2024-09-18T01:19:04.811 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.85759.log: /var/log/ceph/cluster2-client.admin.94766.log: /var/log/ceph/cluster1-client.mirror.35474.log: /var/log/ceph/cluster1-client.mirror.32465.log: /var/log/ceph/cluster2-client.admin.84869.log: /var/log/ceph/cluster1-client.mirror.34320.log: /var/log/ceph/cluster1-client.mirror.120418.log: /var/log/ceph/cluster2-client.admin.61022.log: /var/log/ceph/cluster1-client.admin.49101.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84869.log.gz 2024-09-18T01:19:04.811 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136150.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.61022.log.gz 0.0% 2024-09-18T01:19:04.811 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.49101.log.gz 2024-09-18T01:19:04.811 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.106724.log: /var/log/ceph/cluster1-client.mirror.30932.log: /var/log/ceph/cluster1-client.mirror.78294.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.106724.log.gz 2024-09-18T01:19:04.811 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.65947.log: /var/log/ceph/cluster1-client.mirror.35054.log: /var/log/ceph/cluster1-client.admin.87726.log: /var/log/ceph/cluster2-client.admin.139756.log: /var/log/ceph/cluster1-client.mirror.139137.log: /var/log/ceph/cluster1-client.mirror.109242.log: /var/log/ceph/cluster2-client.admin.140710.log: /var/log/ceph/cluster1-client.mirror.80196.log: /var/log/ceph/cluster1-client.mirror.108001.log: /var/log/ceph/cluster1-client.mirror.69579.log: /var/log/ceph/cluster2-client.admin.128979.log: /var/log/ceph/cluster1-client.mirror.97374.log: /var/log/ceph/cluster2-client.mirror.74760.log: /var/log/ceph/cluster1-client.mirror.37555.log: /var/log/ceph/cluster2-client.mirror.118859.log: /var/log/ceph/cluster1-client.mirror.92180.log: /var/log/ceph/cluster1-client.mirror.46726.log: /var/log/ceph/cluster1-client.mirror.136192.log: /var/log/ceph/cluster1-client.mirror.115223.log: /var/log/ceph/cluster1-client.mirror.127052.log: /var/log/ceph/cluster1-client.mirror.138357.log: /var/log/ceph/cluster1-client.mirror.80236.log: /var/log/ceph/cluster1-client.mirror.82584.log: /var/log/ceph/cluster1-client.mirror.137831.log: /var/log/ceph/cluster1-client.mirror.36212.log: /var/log/ceph/cluster1-client.mirror.79259.log: /var/log/ceph/cluster1-client.mirror.100973.log: /var/log/ceph/cluster1-client.mirror.76263.log: /var/log/ceph/cluster1-client.admin.71267.log: /var/log/ceph/cluster2-client.admin.140281.log: /var/log/ceph/cluster2-client.admin.72960.log: /var/log/ceph/cluster1-client.mirror.31901.log: /var/log/ceph/cluster1-client.mirror.101731.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.71267.log.gz 2024-09-18T01:19:04.811 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.104682.log: /var/log/ceph/cluster1-client.mirror.103294.log: /var/log/ceph/cluster1-client.mirror.94033.log: /var/log/ceph/cluster1-client.mirror.43992.log: /var/log/ceph/cluster1-client.mirror.88001.log: /var/log/ceph/cluster1-client.admin.96654.log: /var/log/ceph/cluster1-client.mirror.121416.log: /var/log/ceph/cluster1-client.admin.42156.log: /var/log/ceph/cluster1-client.admin.117018.log: /var/log/ceph/cluster1-client.mirror.33900.log: /var/log/ceph/cluster1-client.admin.140040.log: /var/log/ceph/cluster1-client.mirror.123167.log: /var/log/ceph/cluster2-client.admin.84262.log: /var/log/ceph/cluster1-client.mirror.103207.log: /var/log/ceph/cluster1-client.mirror.131190.log: /var/log/ceph/cluster1-client.mirror.126317.log: /var/log/ceph/cluster1-client.mirror.115979.log: /var/log/ceph/cluster1-client.mirror.92956.log: /var/log/ceph/cluster1-client.mirror.104849.log: /var/log/ceph/cluster1-client.mirror.138191.log: /var/log/ceph/cluster1-client.mirror.76017.log: /var/log/ceph/cluster2-client.mirror.119637.log: /var/log/ceph/cluster2-client.admin.105789.log: /var/log/ceph/cluster2-client.admin.129469.log: /var/log/ceph/cluster1-client.mirror.48664.log: /var/log/ceph/cluster1-client.mirror.64804.log: /var/log/ceph/cluster1-client.mirror.115854.log: /var/log/ceph/cluster1-client.mirror.108513.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129469.log.gz 2024-09-18T01:19:04.812 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.128102.log: /var/log/ceph/cluster1-client.mirror.65082.log: /var/log/ceph/cluster1-client.admin.86414.log: /var/log/ceph/cluster1-client.mirror.90769.log: /var/log/ceph/cluster1-client.mirror.42086.log: /var/log/ceph/cluster1-client.admin.74137.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.86414.log.gz 2024-09-18T01:19:04.812 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.133113.log: /var/log/ceph/cluster1-client.mirror.54616.log: /var/log/ceph/cluster1-client.admin.83009.log: /var/log/ceph/cluster2-client.admin.106949.log: /var/log/ceph/cluster1-client.mirror.122323.log: /var/log/ceph/cluster1-client.mirror.37263.log: /var/log/ceph/cluster1-client.mirror.67869.log: /var/log/ceph/cluster1-client.mirror.93442.log: /var/log/ceph/cluster1-client.mirror.133997.log: /var/log/ceph/cluster1-client.mirror.31095.log: /var/log/ceph/cluster1-client.mirror.108887.log: /var/log/ceph/cluster1-client.mirror.113856.log: /var/log/ceph/cluster1-client.admin.128060.log: /var/log/ceph/cluster2-client.mirror.117589.log: /var/log/ceph/cluster1-client.mirror.121415.log: /var/log/ceph/cluster1-client.mirror.138767.log: /var/log/ceph/cluster1-client.mirror.112439.log: /var/log/ceph/cluster1-client.admin.117972.log: /var/log/ceph/cluster1-client.mirror.47631.log: /var/log/ceph/cluster1-client.admin.40786.log: /var/log/ceph/cluster1-client.mirror.113221.log: /var/log/ceph/cluster1-client.mirror.36360.log: /var/log/ceph/cluster1-client.mirror.89168.log: /var/log/ceph/cluster1-client.mirror.31970.log: /var/log/ceph/cluster1-client.admin.127456.log: /var/log/ceph/cluster1-client.mirror.97740.log: /var/log/ceph/cluster2-client.admin.29806.log: /var/log/ceph/cluster1-client.admin.75838.log: /var/log/ceph/cluster1-client.admin.120953.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.127456.log.gz 2024-09-18T01:19:04.812 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101353.log: 0.0% 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.75838.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.120953.log.gz 2024-09-18T01:19:04.812 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:04.812 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.94721.log: /var/log/ceph/cluster2-client.admin.96043.log: /var/log/ceph/cluster1-client.mirror.116282.log: /var/log/ceph/cluster2-client.mirror.96385.log: /var/log/ceph/cluster1-client.admin.117268.log: /var/log/ceph/cluster1-client.mirror.32579.log: /var/log/ceph/cluster1-client.mirror.89422.log: /var/log/ceph/cluster1-client.mirror.81706.log: /var/log/ceph/cluster1-client.mirror.77664.log: /var/log/ceph/cluster1-client.admin.71254.log: /var/log/ceph/cluster1-client.mirror.112482.log: /var/log/ceph/cluster1-client.mirror.133070.log: /var/log/ceph/cluster1-client.mirror.53327.log: /var/log/ceph/cluster2-client.admin.117377.log: /var/log/ceph/cluster1-client.mirror.100848.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.71254.log.gz 2024-09-18T01:19:04.812 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.136193.log: /var/log/ceph/cluster1-client.mirror.46938.log: /var/log/ceph/cluster1-client.mirror.68864.log: /var/log/ceph/cluster1-client.mirror.58721.log: /var/log/ceph/cluster1-client.mirror.70396.log: /var/log/ceph/cluster1-client.mirror.81877.log: /var/log/ceph/cluster1-client.mirror.53435.log: /var/log/ceph/cluster1-client.admin.50463.log: /var/log/ceph/cluster2-client.mirror.85075.log: /var/log/ceph/cluster1-client.admin.30225.log: /var/log/ceph/cluster2-client.mirror.107638.log: /var/log/ceph/cluster1-client.mirror.32998.log: /var/log/ceph/cluster1-client.mirror.58112.log: /var/log/ceph/cluster1-client.mirror.59857.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30225.log.gz 2024-09-18T01:19:04.812 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.74345.log: /var/log/ceph/cluster1-client.mirror.103166.log: /var/log/ceph/cluster2-client.mirror.85645.log: /var/log/ceph/cluster1-client.mirror.134630.log: /var/log/ceph/cluster1-client.admin.40101.log: /var/log/ceph/cluster1-client.admin.86727.log: /var/log/ceph/cluster1-client.admin.61387.log: /var/log/ceph/cluster1-client.mirror.124194.log: /var/log/ceph/cluster1-client.mirror.120584.log: /var/log/ceph/cluster1-client.mirror.77289.log: /var/log/ceph/cluster1-client.mirror.56343.log: /var/log/ceph/cluster1-client.mirror.123386.log: /var/log/ceph/cluster2-client.mirror.140904.log: /var/log/ceph/cluster1-client.mirror.63612.log: /var/log/ceph/cluster1-client.mirror.70462.log: /var/log/ceph/cluster1-client.mirror.90283.log: /var/log/ceph/cluster2-client.admin.84878.log: /var/log/ceph/cluster1-client.mirror.136615.log: /var/log/ceph/cluster1-client.admin.121358.log: /var/log/ceph/cluster1-client.admin.63267.log: /var/log/ceph/cluster1-client.mirror.67683.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84878.log.gz 2024-09-18T01:19:04.812 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.116214.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.121358.log.gz 2024-09-18T01:19:04.812 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.66315.log: /var/log/ceph/cluster1-client.mirror.108476.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.116214.log.gz 2024-09-18T01:19:04.812 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.140904.log.gz 2024-09-18T01:19:04.812 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.121108.log: /var/log/ceph/cluster1-client.mirror.44177.log: /var/log/ceph/cluster1-client.mirror.65372.log: /var/log/ceph/cluster1-client.mirror.45841.log: /var/log/ceph/cluster1-client.mirror.110831.log: /var/log/ceph/cluster2-client.mirror.129969.log: /var/log/ceph/cluster2-client.admin.128698.log: /var/log/ceph/cluster1-client.admin.94804.log: /var/log/ceph/cluster1-client.mirror.136407.log: /var/log/ceph/cluster1-client.mirror.53152.log: /var/log/ceph/cluster2-client.admin.95695.log: /var/log/ceph/cluster1-client.mirror.47206.log: /var/log/ceph/cluster1-client.mirror.110447.log: /var/log/ceph/cluster1-client.mirror.35055.log: /var/log/ceph/cluster1-client.admin.41283.log: /var/log/ceph/cluster1-client.mirror.126445.log: /var/log/ceph/cluster2-client.admin.106738.log: /var/log/ceph/cluster1-client.admin.119383.log: /var/log/ceph/cluster1-client.mirror.70673.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.127722.log.gz 2024-09-18T01:19:04.813 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69287.log: /var/log/ceph/cluster1-client.mirror.52575.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.106738.log.gz 2024-09-18T01:19:04.813 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.124681.log: /var/log/ceph/cluster1-client.admin.141793.log: /var/log/ceph/cluster2-client.admin.107296.log: /var/log/ceph/cluster1-client.admin.29618.log: /var/log/ceph/cluster2-client.admin.50793.log: /var/log/ceph/cluster1-client.mirror.121541.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107296.log.gz 2024-09-18T01:19:04.813 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.49792.log: /var/log/ceph/cluster2-client.admin.105752.log: /var/log/ceph/cluster1-client.mirror.102277.log: /var/log/ceph/cluster1-client.mirror.80237.log: /var/log/ceph/cluster1-client.mirror.79435.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.141793.log.gz 2024-09-18T01:19:04.813 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.42308.log: /var/log/ceph/cluster2-client.mirror.29986.log: /var/log/ceph/cluster1-client.mirror.54402.log: /var/log/ceph/cluster1-client.mirror.47038.log: /var/log/ceph/cluster1-client.admin.93971.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42308.log.gz 2024-09-18T01:19:04.813 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85893.log.gz/var/log/ceph/cluster2-client.admin.83944.log: 2024-09-18T01:19:04.813 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138251.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.93971.log.gz 2024-09-18T01:19:04.813 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.37473.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.83944.log.gz 2024-09-18T01:19:04.813 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81496.log: /var/log/ceph/cluster1-client.mirror.66097.log: /var/log/ceph/cluster2-client.admin.84865.log: /var/log/ceph/cluster1-client.mirror.42654.log: /var/log/ceph/cluster1-client.mirror.34572.log: /var/log/ceph/cluster1-client.mirror.92243.log: /var/log/ceph/cluster1-client.admin.30227.log: /var/log/ceph/cluster1-client.mirror.81119.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84865.log.gz 2024-09-18T01:19:04.813 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81749.log: /var/log/ceph/cluster2-client.admin.107172.log: /var/log/ceph/cluster1-client.mirror.33772.log: /var/log/ceph/cluster1-client.mirror.121500.log: /var/log/ceph/cluster1-client.admin.30665.log: /var/log/ceph/cluster1-client.mirror.90557.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107172.log.gz 2024-09-18T01:19:04.813 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.129647.log: /var/log/ceph/cluster1-client.mirror.60200.log: /var/log/ceph/cluster2-client.admin.117784.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30665.log.gz 2024-09-18T01:19:04.813 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.89296.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129647.log.gz 2024-09-18T01:19:04.814 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123712.log: /var/log/ceph/cluster1-client.mirror.133912.log: /var/log/ceph/cluster1-client.mirror.64634.log: /var/log/ceph/cluster1-client.mirror.32359.log: /var/log/ceph/cluster1-client.mirror.59921.log: /var/log/ceph/cluster1-client.mirror.88396.log: /var/log/ceph/cluster1-client.mirror.58575.log: /var/log/ceph/cluster1-client.mirror.69074.log: /var/log/ceph/cluster1-client.mirror.55480.log: /var/log/ceph/cluster1-client.mirror.102149.log: /var/log/ceph/cluster1-client.admin.74050.log: /var/log/ceph/cluster1-client.mirror.31488.log: /var/log/ceph/cluster1-client.mirror.88813.log: /var/log/ceph/cluster1-client.mirror.66400.log: /var/log/ceph/cluster1-client.mirror.124614.log: /var/log/ceph/cluster1-client.mirror.125393.log: /var/log/ceph/cluster1-client.admin.96795.log: /var/log/ceph/cluster1-client.mirror.43633.log: /var/log/ceph/cluster1-client.mirror.88181.log: /var/log/ceph/cluster1-client.mirror.82402.log: /var/log/ceph/cluster2-client.admin.129585.log: /var/log/ceph/cluster1-client.mirror.56047.log: /var/log/ceph/cluster1-client.mirror.49046.log: /var/log/ceph/cluster1-client.mirror.110951.log: /var/log/ceph/cluster1-client.admin.63527.log: /var/log/ceph/cluster1-client.mirror.121867.log: /var/log/ceph/cluster1-client.mirror.135554.log: /var/log/ceph/cluster1-client.mirror.52437.log: /var/log/ceph/cluster1-client.admin.94992.log: /var/log/ceph/cluster2-client.admin.139751.log: /var/log/ceph/cluster1-client.mirror.29635.log: /var/log/ceph/cluster1-client.mirror.69833.log: /var/log/ceph/cluster1-client.admin.85505.log: /var/log/ceph/cluster2-client.admin.50699.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.139751.log.gz 2024-09-18T01:19:04.814 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.45040.log: /var/log/ceph/cluster2-client.admin.61039.log: /var/log/ceph/cluster1-client.mirror.63840.log: /var/log/ceph/cluster1-client.admin.130865.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.50699.log.gz 2024-09-18T01:19:04.814 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.101519.log: /var/log/ceph/cluster1-client.mirror.107773.log: /var/log/ceph/cluster1-client.admin.141766.log: /var/log/ceph/cluster1-client.admin.127448.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.130865.log.gz 2024-09-18T01:19:04.814 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.31240.log: /var/log/ceph/cluster1-client.mirror.79178.log: /var/log/ceph/cluster1-client.mirror.47949.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.127448.log.gz 2024-09-18T01:19:04.814 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.43357.log: /var/log/ceph/cluster1-client.mirror.42421.log: /var/log/ceph/cluster1-client.admin.74023.log: /var/log/ceph/cluster1-client.mirror.90347.log: /var/log/ceph/cluster2-client.admin.128394.log: /var/log/ceph/cluster2-client.mirror.130767.log: /var/log/ceph/cluster2-client.mirror.130177.log: /var/log/ceph/cluster2-client.admin.50078.log: 87.2%/var/log/ceph/cluster2-client.mirror.97069.log: -- replaced with /var/log/ceph/cluster1-client.admin.141766.log.gz 2024-09-18T01:19:04.814 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115748.log: /var/log/ceph/cluster1-client.mirror.54848.log: /var/log/ceph/cluster1-client.mirror.65803.log: /var/log/ceph/cluster1-client.mirror.121910.log: /var/log/ceph/cluster1-client.mirror.132079.log: /var/log/ceph/cluster1-client.admin.38257.log: /var/log/ceph/cluster1-client.mirror.113858.log: /var/log/ceph/cluster1-client.mirror.93231.log: /var/log/ceph/cluster1-client.mirror.81835.log: /var/log/ceph/cluster1-client.mirror.113281.log: /var/log/ceph/cluster1-client.mirror.55564.log: /var/log/ceph/cluster1-client.mirror.76125.log: /var/log/ceph/cluster1-client.mirror.99245.log: /var/log/ceph/cluster1-client.mirror.59501.log: /var/log/ceph/cluster1-client.mirror.136530.log: /var/log/ceph/cluster1-client.mirror.46576.log: /var/log/ceph/cluster2-client.admin.72591.log: /var/log/ceph/cluster1-client.mirror.89086.log: /var/log/ceph/cluster2-client.admin.60920.log: /var/log/ceph/cluster1-client.mirror.123691.log: /var/log/ceph/cluster1-client.mirror.58451.log: /var/log/ceph/cluster1-client.mirror.58450.log: /var/log/ceph/cluster2-client.admin.49669.log: /var/log/ceph/cluster1-client.mirror.44683.log: /var/log/ceph/cluster1-client.mirror.69705.log: /var/log/ceph/cluster1-client.mirror.136405.log: /var/log/ceph/cluster1-client.mirror.135133.log: /var/log/ceph/cluster2-client.admin.62293.log: /var/log/ceph/cluster1-client.mirror.70990.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136194.log.gz 2024-09-18T01:19:04.814 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.41251.log: /var/log/ceph/cluster1-client.admin.52624.log: /var/log/ceph/cluster1-client.mirror.75065.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62293.log.gz 2024-09-18T01:19:04.814 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.75151.log: /var/log/ceph/cluster1-client.admin.127553.log: /var/log/ceph/cluster1-client.mirror.46052.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136406.log.gz 2024-09-18T01:19:04.815 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.115682.log: /var/log/ceph/cluster1-client.mirror.67557.log: /var/log/ceph/cluster2-client.admin.50203.log: /var/log/ceph/cluster1-client.mirror.103208.log: /var/log/ceph/cluster1-client.mirror.101983.log: /var/log/ceph/cluster1-client.admin.60348.log: /var/log/ceph/cluster2-client.admin.128396.log: /var/log/ceph/cluster1-client.mirror.123774.log: /var/log/ceph/cluster1-client.mirror.81079.log: /var/log/ceph/cluster1-client.mirror.86968.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.60348.log.gz 2024-09-18T01:19:04.815 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.68440.log: /var/log/ceph/cluster1-client.admin.83542.log: /var/log/ceph/cluster1-client.mirror.133069.log: /var/log/ceph/cluster1-client.mirror.79774.log: /var/log/ceph/cluster1-client.mirror.122745.log: /var/log/ceph/cluster1-client.mirror.37685.log: /var/log/ceph/cluster1-client.mirror.44201.log: /var/log/ceph/cluster1-client.mirror.42811.log: /var/log/ceph/cluster2-client.admin.48955.log: /var/log/ceph/cluster1-client.mirror.30184.log: /var/log/ceph/cluster1-client.mirror.124680.log: /var/log/ceph/cluster1-client.mirror.120479.log: /var/log/ceph/cluster1-client.mirror.112166.log: /var/log/ceph/cluster1-client.mirror.31511.log: /var/log/ceph/cluster2-client.admin.118393.log: /var/log/ceph/cluster1-client.mirror.45420.log: /var/log/ceph/cluster1-client.mirror.65667.log: /var/log/ceph/cluster1-client.admin.116778.log: /var/log/ceph/cluster1-client.admin.129782.log: /var/log/ceph/cluster1-client.mirror.86909.log: /var/log/ceph/cluster1-client.mirror.111686.log: /var/log/ceph/cluster1-client.mirror.109803.log: /var/log/ceph/cluster1-client.mirror.65455.log: /var/log/ceph/cluster2-client.admin.107291.log: /var/log/ceph/cluster1-client.admin.118092.log: /var/log/ceph/cluster1-client.mirror.132500.log: /var/log/ceph/cluster1-client.mirror.70293.log: /var/log/ceph/cluster2-client.admin.142338.log: /var/log/ceph/cluster1-client.mirror.89714.log: /var/log/ceph/cluster1-client.admin.64544.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107291.log.gz 2024-09-18T01:19:04.815 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.138083.log: /var/log/ceph/cluster2-client.mirror.62743.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142338.log.gz 2024-09-18T01:19:04.815 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.36317.log: /var/log/ceph/cluster1-client.mirror.141974.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.64544.log.gz 2024-09-18T01:19:04.815 INFO:teuthology.orchestra.run.smithi049.stderr: 67.3% -- replaced with /var/log/ceph/cluster1-client.mirror.141974.log.gz 2024-09-18T01:19:04.815 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132648.log: /var/log/ceph/cluster1-client.mirror.66945.log: /var/log/ceph/cluster1-client.mirror.114274.log: /var/log/ceph/cluster1-client.mirror.114699.log: /var/log/ceph/cluster1-client.mirror.126360.log: /var/log/ceph/cluster1-client.mirror.75349.log: /var/log/ceph/cluster1-client.mirror.134457.log: /var/log/ceph/cluster1-client.mirror.110227.log: /var/log/ceph/cluster1-client.mirror.110448.log: /var/log/ceph/cluster2-client.mirror.72894.log: /var/log/ceph/cluster1-client.mirror.134207.log: /var/log/ceph/cluster1-client.admin.48946.log: /var/log/ceph/cluster1-client.mirror.132055.log: /var/log/ceph/cluster1-client.mirror.35199.log: /var/log/ceph/cluster1-client.mirror.111205.log: /var/log/ceph/cluster1-client.mirror.29737.log: /var/log/ceph/cluster2-client.admin.139776.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48946.log.gz 2024-09-18T01:19:04.815 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99541.log: /var/log/ceph/cluster1-client.mirror.37367.log: /var/log/ceph/cluster1-client.mirror.66676.log: /var/log/ceph/cluster2-client.admin.84122.log: /var/log/ceph/cluster2-client.admin.139428.log: /var/log/ceph/cluster1-client.mirror.43356.log: /var/log/ceph/cluster1-client.admin.73708.log: /var/log/ceph/cluster1-client.mirror.113641.log: /var/log/ceph/cluster1-client.mirror.55436.log: /var/log/ceph/cluster1-client.mirror.104049.log: /var/log/ceph/cluster2-client.admin.140706.log: /var/log/ceph/cluster1-client.mirror.126863.log: /var/log/ceph/cluster2-client.mirror.117596.log: /var/log/ceph/cluster1-client.mirror.60136.log: /var/log/ceph/cluster1-client.mirror.36570.log: /var/log/ceph/cluster1-client.mirror.37472.log: /var/log/ceph/cluster1-client.mirror.110741.log: /var/log/ceph/cluster1-client.mirror.78085.log: /var/log/ceph/cluster1-client.admin.97847.log: /var/log/ceph/cluster1-client.mirror.99160.log: /var/log/ceph/cluster1-client.admin.128171.log: /var/log/ceph/cluster1-client.mirror.103503.log: /var/log/ceph/cluster1-client.mirror.67807.log: /var/log/ceph/cluster1-client.mirror.88285.log: /var/log/ceph/cluster1-client.mirror.42980.log: /var/log/ceph/cluster1-client.mirror.99457.log: /var/log/ceph/cluster1-client.admin.73256.log: /var/log/ceph/cluster1-client.mirror.89508.log: /var/log/ceph/cluster1-client.admin.127452.log: /var/log/ceph/cluster1-client.mirror.131253.log: /var/log/ceph/cluster1-client.mirror.42844.log: /var/log/ceph/cluster2-client.mirror.96841.log: /var/log/ceph/cluster1-client.mirror.56134.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.127452.log.gz 2024-09-18T01:19:04.815 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.135529.log: /var/log/ceph/cluster1-client.mirror.137663.log: /var/log/ceph/cluster1-client.mirror.58825.log: /var/log/ceph/cluster1-client.admin.120383.log: /var/log/ceph/cluster1-client.mirror.93292.log: /var/log/ceph/cluster1-client.mirror.54094.log: /var/log/ceph/cluster1-client.mirror.99883.log: /var/log/ceph/cluster1-client.admin.53893.log: /var/log/ceph/cluster2-client.admin.72933.log: /var/log/ceph/cluster2-client.admin.117691.log: /var/log/ceph/cluster1-client.admin.85847.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.53893.log.gz 2024-09-18T01:19:04.816 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.117691.log.gz 2024-09-18T01:19:04.816 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.93084.log: /var/log/ceph/cluster1-client.mirror.115897.log: /var/log/ceph/cluster1-client.mirror.64329.log: /var/log/ceph/cluster1-client.mirror.136823.log: /var/log/ceph/cluster1-client.mirror.64401.log: /var/log/ceph/cluster1-client.mirror.110425.log: /var/log/ceph/cluster1-client.mirror.45314.log: /var/log/ceph/cluster1-client.mirror.69288.log: /var/log/ceph/cluster1-client.mirror.43247.log: /var/log/ceph/cluster1-client.admin.64198.log: /var/log/ceph/cluster1-client.mirror.82084.log: /var/log/ceph/cluster2-client.admin.73019.log: /var/log/ceph/cluster1-client.mirror.53554.log: /var/log/ceph/cluster1-client.admin.140213.log: /var/log/ceph/cluster1-client.mirror.92876.log: /var/log/ceph/cluster1-client.mirror.35836.log: /var/log/ceph/cluster1-client.mirror.136403.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73019.log.gz 2024-09-18T01:19:04.816 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.96996.log: /var/log/ceph/cluster1-client.mirror.69035.log: /var/log/ceph/cluster2-client.mirror.62515.log: /var/log/ceph/cluster1-client.mirror.126295.log: /var/log/ceph/cluster1-client.mirror.86967.log: /var/log/ceph/cluster1-client.mirror.62992.log: /var/log/ceph/cluster1-client.mirror.124341.log: /var/log/ceph/cluster1-client.mirror.31858.log: /var/log/ceph/cluster1-client.mirror.99287.log: /var/log/ceph/cluster1-client.admin.61919.log: /var/log/ceph/cluster1-client.mirror.78629.log: /var/log/ceph/cluster2-client.admin.39390.log: /var/log/ceph/cluster2-client.admin.84260.log: /var/log/ceph/cluster1-client.mirror.77243.log: /var/log/ceph/cluster1-client.mirror.100471.log: /var/log/ceph/cluster1-client.mirror.100010.log: /var/log/ceph/cluster1-client.mirror.93400.log: /var/log/ceph/cluster1-client.mirror.36527.log: /var/log/ceph/cluster1-client.mirror.125664.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84260.log.gz 2024-09-18T01:19:04.816 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.50570.log: /var/log/ceph/cluster1-client.mirror.89297.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.105036.log.gz 2024-09-18T01:19:04.816 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.59161.log: /var/log/ceph/cluster1-client.mirror.137637.log: /var/log/ceph/cluster1-client.admin.120374.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.49715.log.gz 2024-09-18T01:19:04.816 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.131715.log: /var/log/ceph/cluster2-client.admin.94646.log: /var/log/ceph/cluster1-client.mirror.66819.log: /var/log/ceph/cluster1-client.admin.75318.log: /var/log/ceph/cluster1-client.mirror.137032.log: /var/log/ceph/cluster1-client.admin.128187.log: /var/log/ceph/cluster1-client.mirror.88706.log: /var/log/ceph/cluster1-client.admin.84446.log: /var/log/ceph/cluster1-client.mirror.46785.log: /var/log/ceph/cluster1-client.admin.127988.log: /var/log/ceph/cluster1-client.mirror.133111.log: /var/log/ceph/cluster1-client.mirror.114215.log: /var/log/ceph/cluster1-client.mirror.111266.log: /var/log/ceph/cluster1-client.mirror.87560.log: /var/log/ceph/cluster1-client.mirror.108457.log: /var/log/ceph/cluster1-client.mirror.52303.log: /var/log/ceph/cluster1-client.mirror.113071.log: /var/log/ceph/cluster1-client.admin.98352.log: /var/log/ceph/cluster1-client.admin.38589.log: /var/log/ceph/cluster1-client.admin.127995.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.117156.log.gz 2024-09-18T01:19:04.817 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.107096.log: /var/log/ceph/cluster1-client.mirror.90076.log: /var/log/ceph/cluster1-client.mirror.64969.log: /var/log/ceph/cluster1-client.admin.86247.log: /var/log/ceph/cluster1-client.mirror.80279.log: /var/log/ceph/cluster1-client.mirror.70209.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.117349.log.gz 2024-09-18T01:19:04.817 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.64482.log: /var/log/ceph/cluster1-client.mirror.132778.log: /var/log/ceph/cluster1-client.mirror.137160.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.117471.log.gz 2024-09-18T01:19:04.817 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.119222.log: /var/log/ceph/cluster1-client.mirror.67515.log: /var/log/ceph/cluster1-client.mirror.89444.log: /var/log/ceph/cluster1-client.mirror.35452.log: /var/log/ceph/cluster2-client.admin.72496.log: /var/log/ceph/cluster1-client.mirror.54930.log: /var/log/ceph/cluster1-client.admin.120548.log: /var/log/ceph/cluster1-client.mirror.89234.log: /var/log/ceph/cluster1-client.mirror.36780.log: /var/log/ceph/cluster2-client.admin.83834.log: /var/log/ceph/cluster1-client.mirror.100679.log: /var/log/ceph/cluster1-client.mirror.126781.log: /var/log/ceph/cluster1-client.admin.98175.log: /var/log/ceph/cluster1-client.admin.29046.log: /var/log/ceph/cluster1-client.mirror.88816.log: /var/log/ceph/cluster1-client.mirror.69875.log: /var/log/ceph/cluster2-client.admin.95684.log: /var/log/ceph/cluster1-client.mirror.42549.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.98175.log.gz 2024-09-18T01:19:04.817 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.40604.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29046.log.gz 2024-09-18T01:19:04.817 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.61974.log: /var/log/ceph/cluster1-client.mirror.64399.log: /var/log/ceph/cluster1-client.mirror.97446.log: /var/log/ceph/cluster1-client.mirror.37199.log: /var/log/ceph/cluster2-client.admin.39525.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.61974.log.gz 2024-09-18T01:19:04.818 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111101.log: /var/log/ceph/cluster1-client.mirror.78840.log: /var/log/ceph/cluster2-client.admin.72601.log: /var/log/ceph/cluster1-client.mirror.90556.log: /var/log/ceph/cluster1-client.admin.83008.log: /var/log/ceph/cluster1-client.mirror.57986.log: /var/log/ceph/cluster1-client.admin.105063.log: /var/log/ceph/cluster2-client.admin.61773.log: /var/log/ceph/cluster1-client.mirror.81538.log: /var/log/ceph/cluster2-client.admin.84706.log: /var/log/ceph/cluster1-client.admin.64361.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.105063.log.gz 2024-09-18T01:19:04.818 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.81118.log: /var/log/ceph/cluster1-client.mirror.79345.log: /var/log/ceph/cluster2-client.admin.39695.log: /var/log/ceph/cluster1-client.mirror.67765.log: /var/log/ceph/cluster1-client.admin.96882.log: /var/log/ceph/cluster2-client.mirror.129855.log: /var/log/ceph/cluster2-client.admin.117315.log: /var/log/ceph/cluster1-client.mirror.77624.log: /var/log/ceph/cluster2-client.admin.62306.log: /var/log/ceph/cluster1-client.mirror.81414.log: /var/log/ceph/cluster1-client.mirror.99638.log: /var/log/ceph/cluster1-client.mirror.122240.log: /var/log/ceph/cluster1-client.mirror.135559.log: /var/log/ceph/cluster1-client.admin.60658.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62306.log.gz 2024-09-18T01:19:04.818 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.110544.log: /var/log/ceph/cluster1-client.mirror.102450.log: /var/log/ceph/cluster1-client.mirror.31023.log: /var/log/ceph/cluster2-client.admin.39372.log: /var/log/ceph/cluster2-client.admin.107171.log: /var/log/ceph/cluster1-client.mirror.47990.log: /var/log/ceph/cluster1-client.mirror.80951.log: /var/log/ceph/cluster2-client.admin.95949.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.39372.log.gz 2024-09-18T01:19:04.818 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.132603.log: /var/log/ceph/cluster2-client.admin.129283.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.107171.log.gz 2024-09-18T01:19:04.818 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87918.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.95949.log.gz 2024-09-18T01:19:04.819 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.84556.log: /var/log/ceph/cluster2-client.mirror.118973.log: /var/log/ceph/cluster2-client.mirror.140654.log: /var/log/ceph/cluster1-client.mirror.31061.log: /var/log/ceph/cluster1-client.mirror.81959.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84556.log.gz 2024-09-18T01:19:04.819 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80407.log: /var/log/ceph/cluster1-client.mirror.137453.log: /var/log/ceph/cluster1-client.admin.85391.log: /var/log/ceph/cluster2-client.admin.49888.log: /var/log/ceph/cluster1-client.mirror.41913.log: /var/log/ceph/cluster1-client.admin.127621.log: /var/log/ceph/cluster1-client.mirror.80068.log: /var/log/ceph/cluster1-client.mirror.126106.log: /var/log/ceph/cluster2-client.admin.140686.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.127621.log.gz 2024-09-18T01:19:04.819 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.51569.log: /var/log/ceph/cluster2-client.mirror.40127.log: /var/log/ceph/cluster1-client.mirror.123211.log: /var/log/ceph/cluster2-client.admin.139232.log: /var/log/ceph/cluster1-client.mirror.104343.log: /var/log/ceph/cluster1-client.mirror.126235.log: /var/log/ceph/cluster1-client.mirror.121700.log: /var/log/ceph/cluster1-client.mirror.138002.log: /var/log/ceph/cluster1-client.mirror.93875.log: /var/log/ceph/cluster1-client.mirror.125897.log: /var/log/ceph/cluster1-client.admin.108135.log: /var/log/ceph/cluster1-client.mirror.42240.log: /var/log/ceph/cluster1-client.mirror.34215.log: /var/log/ceph/cluster1-client.mirror.105442.log: /var/log/ceph/cluster2-client.admin.50710.log: /var/log/ceph/cluster1-client.mirror.126210.log: /var/log/ceph/cluster1-client.mirror.136110.log: /var/log/ceph/cluster1-client.mirror.32577.log: /var/log/ceph/cluster1-client.admin.105471.log: /var/log/ceph/cluster1-client.mirror.58785.log: /var/log/ceph/cluster1-client.admin.60502.log: /var/log/ceph/cluster1-client.mirror.100514.log: /var/log/ceph/cluster1-client.mirror.79859.log: /var/log/ceph/cluster1-client.mirror.32681.log: /var/log/ceph/cluster1-client.mirror.114970.log: /var/log/ceph/cluster1-client.admin.93779.log: /var/log/ceph/cluster2-osd.0.log: /var/log/ceph/cluster1-client.mirror.102787.log: /var/log/ceph/cluster1-client.mirror.55059.log: /var/log/ceph/cluster2-client.admin.117698.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.93779.log.gz 2024-09-18T01:19:04.819 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.32268.log: /var/log/ceph/cluster1-client.mirror.109829.log: /var/log/ceph/cluster1-client.admin.84172.log: /var/log/ceph/cluster1-client.admin.82618.log: /var/log/ceph/cluster1-client.mirror.62422.log: /var/log/ceph/cluster1-client.admin.97224.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.82618.log.gz 2024-09-18T01:19:04.819 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.72228.log: /var/log/ceph/cluster1-client.mirror.70843.log: /var/log/ceph/cluster2-client.admin.140683.log: /var/log/ceph/cluster1-client.mirror.79387.log: /var/log/ceph/cluster2-client.mirror.40470.log: /var/log/ceph/cluster1-client.mirror.127158.log: /var/log/ceph/cluster1-client.mirror.115853.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.140683.log.gz 2024-09-18T01:19:04.819 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.139561.log: /var/log/ceph/cluster1-client.mirror.79094.log: /var/log/ceph/cluster1-client.mirror.78295.log: /var/log/ceph/cluster1-client.mirror.37641.log: /var/log/ceph/cluster1-client.mirror.65414.log: /var/log/ceph/cluster1-client.mirror.67096.log: /var/log/ceph/cluster2-client.mirror.130083.log: /var/log/ceph/cluster1-client.mirror.92032.log: /var/log/ceph/cluster1-client.mirror.58239.log: /var/log/ceph/cluster1-client.mirror.135342.log: /var/log/ceph/cluster1-client.mirror.35098.log: /var/log/ceph/cluster1-client.mirror.68017.log: /var/log/ceph/cluster1-client.mirror.69832.log: /var/log/ceph/cluster2-client.mirror.51776.log: /var/log/ceph/cluster1-client.mirror.103083.log: /var/log/ceph/cluster1-client.mirror.124470.log: /var/log/ceph/cluster1-client.mirror.37052.log: /var/log/ceph/cluster1-client.admin.38585.log: /var/log/ceph/cluster1-client.admin.38548.log: /var/log/ceph/cluster1-client.mirror.126505.log: /var/log/ceph/cluster1-client.mirror.91616.log: /var/log/ceph/cluster1-client.mirror.56006.log: /var/log/ceph/cluster1-client.admin.108390.log: /var/log/ceph/cluster1-client.mirror.133027.log: /var/log/ceph/cluster1-client.mirror.36462.log: /var/log/ceph/cluster1-client.mirror.67895.log: /var/log/ceph/cluster1-client.mirror.104134.log: /var/log/ceph/cluster1-client.mirror.127115.log: /var/log/ceph/cluster2-client.admin.140748.log: /var/log/ceph/cluster1-client.mirror.80532.log: /var/log/ceph/cluster1-client.mirror.104513.log: /var/log/ceph/cluster1-client.mirror.75064.log: /var/log/ceph/cluster2-client.admin.38799.log: /var/log/ceph/cluster1-client.mirror.91844.log: /var/log/ceph/cluster2-client.admin.73016.log: /var/log/ceph/cluster1-client.admin.62556.log: /var/log/ceph/cluster1-client.mirror.46827.log: /var/log/ceph/cluster1-client.admin.105572.log: /var/log/ceph/cluster1-client.admin.106742.log: /var/log/ceph/cluster1-client.mirror.121258.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73016.log.gz 2024-09-18T01:19:04.819 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.67348.log: /var/log/ceph/cluster1-client.mirror.141401.log: /var/log/ceph/cluster1-client.mirror.58089.log: /var/log/ceph/cluster1-client.mirror.33164.log: /var/log/ceph/cluster1-client.mirror.36950.log: /var/log/ceph/cluster1-client.mirror.135361.log: /var/log/ceph/cluster1-client.mirror.116218.log: /var/log/ceph/cluster1-client.mirror.70419.log: 67.8% -- replaced with /var/log/ceph/cluster1-client.mirror.141401.log.gz 2024-09-18T01:19:04.819 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.123479.log: /var/log/ceph/cluster1-client.mirror.43784.log: /var/log/ceph/cluster1-client.mirror.33417.log: /var/log/ceph/cluster2-client.mirror.40376.log: /var/log/ceph/cluster1-client.mirror.41231.log: /var/log/ceph/cluster1-client.mirror.116215.log: /var/log/ceph/cluster1-client.mirror.44834.log: /var/log/ceph/cluster1-client.mirror.103375.log: /var/log/ceph/cluster1-client.mirror.136402.log: /var/log/ceph/cluster1-client.mirror.68313.log: /var/log/ceph/cluster1-client.mirror.43993.log: /var/log/ceph/cluster1-client.mirror.32745.log: /var/log/ceph/cluster1-client.mirror.33374.log: /var/log/ceph/cluster1-client.admin.131132.log: /var/log/ceph/cluster1-client.mirror.56089.log: /var/log/ceph/cluster1-client.mirror.92394.log: /var/log/ceph/cluster1-client.mirror.30015.log: /var/log/ceph/cluster1-client.mirror.67179.log: /var/log/ceph/cluster1-client.mirror.76193.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.131132.log.gz 2024-09-18T01:19:04.819 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.118725.log: /var/log/ceph/cluster1-client.admin.128165.log: /var/log/ceph/cluster1-client.mirror.103628.log: /var/log/ceph/cluster1-client.admin.83897.log: /var/log/ceph/cluster1-client.mirror.123848.log: /var/log/ceph/cluster1-client.mirror.133029.log: /var/log/ceph/cluster1-client.mirror.30869.log: /var/log/ceph/cluster1-client.mirror.114632.log: /var/log/ceph/cluster1-client.mirror.135513.log: /var/log/ceph/cluster1-client.mirror.90221.log: /var/log/ceph/cluster1-client.mirror.44833.log: /var/log/ceph/cluster1-client.admin.130694.log: /var/log/ceph/cluster1-client.mirror.126150.log: /var/log/ceph/cluster1-client.mirror.55837.log: /var/log/ceph/cluster1-client.mirror.115474.log: /var/log/ceph/cluster1-client.mirror.122934.log: /var/log/ceph/cluster1-client.mirror.121190.log: /var/log/ceph/cluster1-client.mirror.33730.log: /var/log/ceph/cluster1-client.mirror.134297.log: /var/log/ceph/cluster1-client.mirror.36085.log: /var/log/ceph/cluster2-client.admin.107035.log: /var/log/ceph/cluster1-client.admin.116787.log: /var/log/ceph/cluster1-client.mirror.40283.log: /var/log/ceph/cluster2-client.admin.94094.log: /var/log/ceph/cluster2-client.admin.128759.log: /var/log/ceph/cluster2-client.admin.117704.log: /var/log/ceph/cluster1-client.mirror.36885.log: /var/log/ceph/cluster1-client.mirror.30752.log: /var/log/ceph/cluster1-client.mirror.134079.log: /var/log/ceph/cluster1-client.mirror.55922.log: /var/log/ceph/cluster1-client.admin.87474.log: /var/log/ceph/cluster1-client.mirror.125789.log: /var/log/ceph/cluster1-client.admin.30830.log: /var/log/ceph/cluster2-client.mirror.107128.log: /var/log/ceph/cluster2-client.admin.38937.log: /var/log/ceph/cluster1-client.mirror.101404.log: /var/log/ceph/cluster1-client.mirror.37368.log: /var/log/ceph/cluster1-client.mirror.134647.log: /var/log/ceph/cluster1-client.mirror.47359.log: /var/log/ceph/cluster1-client.mirror.64636.log: /var/log/ceph/cluster1-client.mirror.37516.log: /var/log/ceph/cluster1-client.mirror.114843.log: /var/log/ceph/cluster1-client.mirror.80740.log: /var/log/ceph/cluster1-client.mirror.103883.log: /var/log/ceph/cluster1-client.mirror.68737.log: /var/log/ceph/cluster1-client.mirror.66925.log: /var/log/ceph/cluster1-client.mirror.43994.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.84683.log.gz 2024-09-18T01:19:04.819 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.83712.log: /var/log/ceph/cluster1-client.mirror.44958.log: /var/log/ceph/cluster1-client.mirror.124679.log: /var/log/ceph/cluster2-client.mirror.116757.log: /var/log/ceph/cluster1-client.mirror.33059.log: /var/log/ceph/cluster1-client.mirror.111204.log: /var/log/ceph/cluster1-client.mirror.120422.log: /var/log/ceph/cluster1-client.mirror.31448.log: /var/log/ceph/cluster1-client.mirror.135899.log: /var/log/ceph/cluster1-client.mirror.42633.log: /var/log/ceph/cluster2-client.admin.129530.log: /var/log/ceph/cluster2-client.admin.62284.log: /var/log/ceph/cluster1-client.mirror.91359.log: /var/log/ceph/cluster2-client.admin.116731.log: /var/log/ceph/cluster1-client.mirror.102065.log: /var/log/ceph/cluster1-client.mirror.76896.log: /var/log/ceph/cluster1-client.mirror.111810.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.62284.log.gz 2024-09-18T01:19:04.819 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.142390.log: /var/log/ceph/cluster1-client.mirror.99840.log: /var/log/ceph/cluster1-client.mirror.59581.log: /var/log/ceph/cluster1-client.admin.38256.log: /var/log/ceph/cluster1-client.mirror.88329.log: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107410.log.gz 0.0% 2024-09-18T01:19:04.819 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.142390.log.gz 2024-09-18T01:19:04.820 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.63794.log: /var/log/ceph/cluster1-client.mirror.136780.log: /var/log/ceph/cluster1-client.mirror.137370.log: /var/log/ceph/cluster1-client.mirror.132824.log: /var/log/ceph/cluster1-client.mirror.65328.log: /var/log/ceph/cluster2-client.admin.117693.log: /var/log/ceph/cluster1-client.mirror.99050.log: /var/log/ceph/cluster1-client.mirror.65373.log: /var/log/ceph/cluster1-client.mirror.131716.log: /var/log/ceph/cluster2-client.admin.61305.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.117693.log.gz 2024-09-18T01:19:04.820 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111682.log: /var/log/ceph/cluster1-client.mirror.65715.log: /var/log/ceph/cluster2-client.mirror.62155.log: /var/log/ceph/cluster2-client.mirror.39384.log: /var/log/ceph/cluster2-client.admin.129265.log: /var/log/ceph/cluster1-client.mirror.42422.log: /var/log/ceph/cluster1-client.admin.49356.log: /var/log/ceph/cluster1-client.admin.75472.log: /var/log/ceph/cluster2-client.admin.40066.log: /var/log/ceph/cluster1-client.mirror.34888.log: /var/log/ceph/cluster2-client.admin.39474.log: /var/log/ceph/cluster1-client.mirror.123901.log: /var/log/ceph/cluster1-client.mirror.43950.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.75472.log.gz 2024-09-18T01:19:04.820 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.80573.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.40066.log.gz 2024-09-18T01:19:04.820 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.admin.40558.log: /var/log/ceph/cluster1-client.mirror.33837.log: /var/log/ceph/cluster1-client.admin.61353.log: /var/log/ceph/cluster2-client.mirror.130311.log: /var/log/ceph/cluster2-client.admin.117883.log: /var/log/ceph/cluster1-client.mirror.36423.log: /var/log/ceph/cluster1-client.mirror.102743.log: /var/log/ceph/cluster1-client.mirror.92601.log: /var/log/ceph/cluster1-client.mirror.79944.log: /var/log/ceph/cluster1-client.mirror.81370.log: /var/log/ceph/cluster1-client.mirror.126780.log: /var/log/ceph/cluster1-client.mirror.86704.log: /var/log/ceph/cluster1-client.mirror.79093.log: /var/log/ceph/cluster1-client.mirror.116522.log: /var/log/ceph/cluster1-client.mirror.110355.log: /var/log/ceph/cluster1-client.mirror.65843.log: /var/log/ceph/cluster2-client.admin.84896.log: /var/log/ceph/cluster1-client.mirror.54403.log: /var/log/ceph/cluster1-client.mirror.137452.log: /var/log/ceph/cluster1-client.mirror.42114.log: /var/log/ceph/cluster1-client.mirror.56679.log: /var/log/ceph/cluster1-client.mirror.100617.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84896.log.gz 2024-09-18T01:19:04.820 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.85303.log: /var/log/ceph/cluster1-client.mirror.36672.log: /var/log/ceph/cluster1-client.admin.49113.log: /var/log/ceph/cluster1-client.admin.142041.log: /var/log/ceph/cluster1-client.mirror.111034.log: /var/log/ceph/cluster1-client.mirror.59455.log: /var/log/ceph/cluster1-client.mirror.68865.log: /var/log/ceph/cluster1-client.admin.109891.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.49113.log.gz 2024-09-18T01:19:04.820 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.87759.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.142041.log.gz 2024-09-18T01:19:04.820 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.109891.log.gz 2024-09-18T01:19:04.820 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.91404.log: /var/log/ceph/cluster1-client.mirror.76636.log: /var/log/ceph/cluster2-client.admin.83763.log: /var/log/ceph/cluster1-client.mirror.80824.log: /var/log/ceph/cluster1-client.mirror.82190.log: /var/log/ceph/cluster2-client.admin.50741.log: /var/log/ceph/cluster1-client.mirror.92666.log: /var/log/ceph/cluster1-client.mirror.88129.log: /var/log/ceph/cluster1-client.mirror.67681.log: /var/log/ceph/cluster1-client.mirror.56425.log: /var/log/ceph/cluster1-client.mirror.68019.log: /var/log/ceph/cluster1-client.mirror.71388.log: /var/log/ceph/cluster1-client.mirror.102320.log: /var/log/ceph/cluster1-client.mirror.111328.log: /var/log/ceph/cluster1-client.mirror.42204.log: /var/log/ceph/cluster1-client.mirror.75880.log: /var/log/ceph/cluster1-client.mirror.87247.log: /var/log/ceph/cluster1-client.mirror.56424.log: /var/log/ceph/cluster1-client.admin.118672.log: /var/log/ceph/cluster1-client.mirror.54763.log: /var/log/ceph/cluster1-client.mirror.87225.log: /var/log/ceph/cluster1-client.admin.49281.log: /var/log/ceph/cluster1-client.mirror.43417.log: /var/log/ceph/cluster1-client.admin.141649.log: /var/log/ceph/cluster1-client.mirror.68501.log: /var/log/ceph/cluster1-client.mirror.64684.log: /var/log/ceph/cluster1-client.mirror.104301.log: /var/log/ceph/cluster1-client.mirror.113789.log: /var/log/ceph/cluster2-client.admin.39985.log: /var/log/ceph/cluster1-client.admin.118080.log: /var/log/ceph/cluster1-client.mirror.114440.log: /var/log/ceph/cluster1-client.admin.38393.log: /var/log/ceph/cluster2-client.mirror.74304.log: /var/log/ceph/cluster1-client.admin.52356.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.141649.log.gz 2024-09-18T01:19:04.820 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.122242.log: /var/log/ceph/cluster1-client.mirror.65587.log: /var/log/ceph/cluster2-client.admin.61020.log: /var/log/ceph/cluster2-client.admin.129310.log: /var/log/ceph/cluster1-client.mirror.102874.log: /var/log/ceph/cluster2-client.mirror.62951.log: /var/log/ceph/cluster1-client.mirror.67599.log: /var/log/ceph/cluster1-client.mirror.124425.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.61020.log.gz 2024-09-18T01:19:04.820 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.51320.log: /var/log/ceph/cluster1-client.mirror.115898.log: /var/log/ceph/cluster1-client.mirror.91552.log: /var/log/ceph/cluster1-client.mirror.138515.log: /var/log/ceph/cluster1-client.mirror.99926.log: /var/log/ceph/cluster1-client.mirror.53105.log: /var/log/ceph/cluster1-client.admin.51247.log: /var/log/ceph/cluster1-client.mirror.111956.log: /var/log/ceph/cluster2-client.mirror.129207.log: /var/log/ceph/cluster1-client.admin.61496.log: /var/log/ceph/cluster1-client.mirror.76436.log: /var/log/ceph/cluster1-client.mirror.32932.log: /var/log/ceph/cluster1-client.mirror.89169.log: /var/log/ceph/cluster2-client.admin.39397.log: /var/log/ceph/cluster2-client.mirror.85209.log: /var/log/ceph/cluster1-client.admin.105072.log: /var/log/ceph/cluster1-client.mirror.104468.log: /var/log/ceph/cluster1-client.admin.128312.log: /var/log/ceph/cluster1-client.mirror.67471.log: /var/log/ceph/cluster1-client.admin.97397.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.105072.log.gz 2024-09-18T01:19:04.821 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.38912.log.gz 2024-09-18T01:19:04.821 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.109396.log: /var/log/ceph/cluster1-client.mirror.59710.log: /var/log/ceph/cluster1-client.mirror.66057.log: /var/log/ceph/cluster1-client.mirror.87072.log: /var/log/ceph/cluster1-client.admin.50321.log: /var/log/ceph/cluster1-client.mirror.125896.log: /var/log/ceph/cluster2-osd.2.log: /var/log/ceph/cluster1-client.admin.116208.log: /var/log/ceph/cluster1-client.mirror.32954.log: /var/log/ceph/cluster1-client.mirror.75451.log: /var/log/ceph/cluster1-client.mirror.58870.log: /var/log/ceph/cluster1-client.mirror.108583.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.116208.log.gz 2024-09-18T01:19:04.821 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.95423.log.gz 2024-09-18T01:19:04.821 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.47841.log: /var/log/ceph/cluster2-client.admin.106741.log: /var/log/ceph/cluster1-client.mirror.87420.log: /var/log/ceph/cluster1-client.mirror.82168.log: /var/log/ceph/cluster1-client.admin.110101.log: /var/log/ceph/cluster1-client.mirror.135900.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.110101.log.gz 2024-09-18T01:19:04.821 INFO:teuthology.orchestra.run.smithi049.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.106741.log.gz 2024-09-18T01:19:04.821 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.83706.log: /var/log/ceph/cluster1-client.mirror.59415.log: /var/log/ceph/cluster2-client.admin.129262.log: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.39882.log.gz 2024-09-18T01:19:04.821 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.137874.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.83706.log.gz 2024-09-18T01:19:04.821 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.111541.log: /var/log/ceph/cluster2-client.admin.128963.log: /var/log/ceph/cluster1-client.mirror.47906.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.129262.log.gz 2024-09-18T01:19:04.821 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.admin.117702.log: /var/log/ceph/cluster1-client.mirror.92663.log: /var/log/ceph/cluster1-client.mirror.76645.log: /var/log/ceph/cluster1-client.mirror.111957.log: /var/log/ceph/cluster1-client.mirror.127264.log: /var/log/ceph/cluster2-client.mirror.130405.log: /var/log/ceph/cluster1-client.mirror.103417.log: /var/log/ceph/cluster2-client.mirror.62495.log: /var/log/ceph/cluster1-client.mirror.67053.log: /var/log/ceph/cluster1-client.mirror.81245.log: /var/log/ceph/cluster2-client.admin.142156.log: /var/log/ceph/cluster1-client.mirror.32996.log: /var/log/ceph/cluster1-client.mirror.37412.log: /var/log/ceph/cluster1-client.mirror.109765.log: /var/log/ceph/cluster1-client.admin.74878.log: /var/log/ceph/cluster1-client.mirror.133742.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142156.log.gz 2024-09-18T01:19:04.821 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.140924.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.74878.log.gz 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.69874.log: /var/log/ceph/cluster1-client.admin.94517.log: /var/log/ceph/cluster1-client.mirror.45211.log: /var/log/ceph/cluster1-client.mirror.100133.log: /var/log/ceph/cluster1-client.mirror.91109.log: /var/log/ceph/cluster1-client.admin.31308.log: /var/log/ceph/cluster1-client.mirror.82423.log: /var/log/ceph/cluster1-client.mirror.110950.log: /var/log/ceph/cluster1-client.admin.96453.log: /var/log/ceph/cluster1-client.mirror.79816.log: /var/log/ceph/cluster2-client.admin.129594.log: /var/log/ceph/cluster2-client.mirror.86007.log: /var/log/ceph/cluster1-client.admin.130607.log: /var/log/ceph/cluster1-client.admin.29246.log: /var/log/ceph/cluster1-client.mirror.99330.log: /var/log/ceph/cluster2-client.admin.140459.log: /var/log/ceph/cluster1-client.mirror.112732.log: 0.0%/var/log/ceph/cluster2-client.admin.118438.log: -- replaced with /var/log/ceph/cluster1-client.admin.29246.log.gz 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.99968.log: /var/log/ceph/cluster1-client.mirror.57392.log: /var/log/ceph/cluster1-client.mirror.54319.log: /var/log/ceph/cluster2-client.admin.73673.log: /var/log/ceph/cluster1-client.mirror.74837.log: /var/log/ceph/cluster1-client.mirror.66099.log: /var/log/ceph/cluster1-client.mirror.122451.log: /var/log/ceph/cluster1-client.mirror.70544.log: /var/log/ceph/cluster1-client.admin.117053.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.73673.log.gz 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster1-client.mirror.114277.log: /var/log/ceph/cluster1-client.mirror.57032.log: /var/log/ceph/cluster1-client.mirror.68101.log: /var/log/ceph/cluster1-client.mirror.78219.log: /var/log/ceph/cluster1-client.mirror.131428.log: /var/log/ceph/cluster1-client.mirror.137580.log: /var/log/ceph/cluster2-client.mirror.107866.log: /var/log/ceph/cluster1-client.admin.108276.log: /var/log/ceph/cluster1-client.mirror.66014.log: /var/log/ceph/cluster1-client.mirror.119847.log: /var/log/ceph/cluster1-client.mirror.41492.log: /var/log/ceph/cluster2-client.admin.29823.log: /var/log/ceph/cluster1-client.mirror.109491.log: /var/log/ceph/cluster2-client.admin.73063.log: /var/log/ceph/cluster1-client.mirror.81076.log: /var/log/ceph/cluster2-client.admin.61677.log: /var/log/ceph/cluster1-client.mirror.131228.log: /var/log/ceph/cluster1-client.mirror.44894.log: /var/log/ceph/cluster1-client.mirror.115327.log: /var/log/ceph/cluster1-client.mirror.77833.log: /var/log/ceph/cluster1-client.mirror.75203.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.61677.log.gz 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr:/var/log/ceph/cluster2-client.mirror.50257.log: /var/log/ceph/cluster1-client.admin.93851.log: /var/log/ceph/cluster1-client.mirror.56845.log: /var/log/ceph/cluster1-client.mirror.108621.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.38613.log.gz 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137581.log.gz 92.1% 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.38694.log.gz 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85779.log.gz 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68822.log.gz 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.129215.log.gz 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.140300.log.gz 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.62102.log.gz 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108208.log.gz 2024-09-18T01:19:04.822 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.79346.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70042.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137033.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93024.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40147.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77707.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.79051.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.98837.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101438.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100724.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133575.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.60578.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.49006.log.gz 2024-09-18T01:19:04.823 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.38935.log.gz 2024-09-18T01:19:04.824 INFO:teuthology.orchestra.run.smithi049.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.63878.log.gz 2024-09-18T01:19:04.824 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97183.log.gz 2024-09-18T01:19:04.824 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44327.log.gz 2024-09-18T01:19:04.824 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81580.log.gz 2024-09-18T01:19:04.824 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 96.3% -- replaced with /var/log/ceph/cluster1-client.mirror.101773.log.gz 2024-09-18T01:19:04.824 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.73982.log.gz 2024-09-18T01:19:04.824 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97297.log.gz 2024-09-18T01:19:04.824 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54182.log.gz 2024-09-18T01:19:04.824 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112272.log.gz 2024-09-18T01:19:04.824 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.73936.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.140538.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.51046.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.93857.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122662.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69494.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.71936.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70927.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.121085.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.140167.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.118290.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56761.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40832.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.40215.log.gz 2024-09-18T01:19:04.825 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.49468.log.gz 89.9% 2024-09-18T01:19:04.826 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.67659.log.gz 2024-09-18T01:19:04.826 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.83396.log.gz 2024-09-18T01:19:04.826 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.84354.log.gz 2024-09-18T01:19:04.826 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.82735.log.gz 2024-09-18T01:19:04.826 INFO:teuthology.orchestra.run.smithi049.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.97893.log.gz 2024-09-18T01:19:04.826 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.105599.log.gz 2024-09-18T01:19:04.826 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108322.log.gz 2024-09-18T01:19:04.826 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96271.log.gz 2024-09-18T01:19:04.826 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59791.log.gz 2024-09-18T01:19:04.826 INFO:teuthology.orchestra.run.smithi049.stderr: 67.5% 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.62764.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.120286.log.gz 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.105168.log.gz 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.129471.log.gz -- replaced with /var/log/ceph/cluster2-client.admin.106957.log.gz 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.admin.128505.log.gz 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.95257.log.gz 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108094.log.gz 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.126316.log.gz 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115599.log.gz 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130425.log.gz 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.39936.log.gz 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63427.log.gz 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119201.log.gz 2024-09-18T01:19:04.827 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.85458.log.gz 2024-09-18T01:19:04.828 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.82935.log.gz 2024-09-18T01:19:04.828 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.93356.log.gz 2024-09-18T01:19:04.828 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster2-client.admin.61276.log.gz 2024-09-18T01:19:04.828 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53488.log.gz 2024-09-18T01:19:04.828 INFO:teuthology.orchestra.run.smithi049.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.62723.log.gz 2024-09-18T01:19:04.828 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.140924.log.gz 2024-09-18T01:19:04.828 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.134139.log.gz 2024-09-18T01:19:04.828 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137034.log.gz 2024-09-18T01:19:04.828 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90348.log.gz 2024-09-18T01:19:04.828 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.38394.log.gz 2024-09-18T01:19:04.828 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59456.log.gz 2024-09-18T01:19:04.828 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 91.6% -- replaced with /var/log/ceph/cluster1-client.mirror.125941.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.138888.log.gz 2024-09-18T01:19:04.829 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:04.829 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119295.log.gz 2024-09-18T01:19:04.829 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.134037.log.gz 2024-09-18T01:19:04.829 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73754.log.gz 2024-09-18T01:19:04.829 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63199.log.gz 2024-09-18T01:19:04.829 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90896.log.gz 2024-09-18T01:19:04.829 INFO:teuthology.orchestra.run.smithi049.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.admin.38926.log.gz 2024-09-18T01:19:04.829 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.105986.log.gz 2024-09-18T01:19:04.829 INFO:teuthology.orchestra.run.smithi049.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.admin.128179.log.gz 2024-09-18T01:19:04.829 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57732.log.gz 2024-09-18T01:19:04.829 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67347.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49803.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.85277.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51548.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85665.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.140193.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70127.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 67.5% -- replaced with /var/log/ceph/cluster1-client.mirror.41195.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.120462.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110698.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.83924.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.admin.130265.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.admin.95395.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.107364.log.gz 2024-09-18T01:19:04.830 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32069.log.gz 2024-09-18T01:19:04.831 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51206.log.gz 2024-09-18T01:19:04.831 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107524.log.gz 2024-09-18T01:19:04.831 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.118585.log.gz 2024-09-18T01:19:04.831 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89801.log.gz 2024-09-18T01:19:04.831 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.118246.log.gz 2024-09-18T01:19:04.831 INFO:teuthology.orchestra.run.smithi049.stderr: 18.2% -- replaced with /var/log/ceph/cluster1-client.admin.64546.log.gz 2024-09-18T01:19:04.831 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111640.log.gz 2024-09-18T01:19:04.831 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.139285.log.gz 2024-09-18T01:19:04.831 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67305.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119315.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34635.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: 57.2% -- replaced with /var/log/ceph/cluster1-client.mirror.51455.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69918.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.110405.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108302.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.140480.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.50587.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99037.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.94956.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79563.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.121728.log.gz 2024-09-18T01:19:04.832 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59077.log.gz 2024-09-18T01:19:04.833 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.105479.log.gz 2024-09-18T01:19:04.833 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.72120.log.gz 2024-09-18T01:19:04.833 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132750.log.gz 2024-09-18T01:19:04.833 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.119873.log.gz 2024-09-18T01:19:04.833 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59711.log.gz 2024-09-18T01:19:04.833 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.64196.log.gz 2024-09-18T01:19:04.833 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.87753.log.gz 2024-09-18T01:19:04.833 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.81204.log.gz 2024-09-18T01:19:04.833 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66443.log.gz 2024-09-18T01:19:04.833 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67683.log.gz 2024-09-18T01:19:04.833 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67681.log.gz 2024-09-18T01:19:04.833 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100617.log.gz 2024-09-18T01:19:04.834 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.113492.log.gz 2024-09-18T01:19:04.834 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123901.log.gz 2024-09-18T01:19:04.834 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123712.log.gz 89.8% 2024-09-18T01:19:04.834 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.123774.log.gz 2024-09-18T01:19:04.842 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58995.log.gz 2024-09-18T01:19:04.843 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58994.log.gz 2024-09-18T01:19:04.844 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.64401.log.gz 2024-09-18T01:19:04.845 INFO:teuthology.orchestra.run.smithi049.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.64399.log.gz 2024-09-18T01:19:04.850 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56425.log.gz 2024-09-18T01:19:04.850 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56424.log.gz 2024-09-18T01:19:04.850 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56467.log.gz 2024-09-18T01:19:04.853 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65803.log.gz 2024-09-18T01:19:04.892 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63313.log.gz 2024-09-18T01:19:04.943 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.95357.log.gz 2024-09-18T01:19:04.943 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.admin.72623.log.gz 2024-09-18T01:19:04.945 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.74930.log.gz 2024-09-18T01:19:04.948 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33795.log.gz 2024-09-18T01:19:04.949 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33794.log.gz 2024-09-18T01:19:04.958 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92180.log.gz 2024-09-18T01:19:04.960 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88792.log.gz 2024-09-18T01:19:04.961 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88813.log.gz 2024-09-18T01:19:04.964 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78883.log.gz 2024-09-18T01:19:04.964 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78840.log.gz 2024-09-18T01:19:04.964 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78884.log.gz 2024-09-18T01:19:04.966 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91276.log.gz 2024-09-18T01:19:04.966 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91319.log.gz 2024-09-18T01:19:04.971 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.140542.log.gz 2024-09-18T01:19:04.973 INFO:teuthology.orchestra.run.smithi049.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.admin.118239.log.gz 2024-09-18T01:19:04.973 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.42204.log.gz 2024-09-18T01:19:04.974 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42240.log.gz 2024-09-18T01:19:04.980 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89926.log.gz 2024-09-18T01:19:04.980 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89927.log.gz 2024-09-18T01:19:04.981 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43129.log.gz 2024-09-18T01:19:04.984 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81037.log.gz 2024-09-18T01:19:05.046 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126150.log.gz 2024-09-18T01:19:05.081 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.94458.log.gz 2024-09-18T01:19:05.085 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79136.log.gz 2024-09-18T01:19:05.094 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.60200.log.gz 2024-09-18T01:19:05.114 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55141.log.gz 2024-09-18T01:19:05.217 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111076.log.gz 2024-09-18T01:19:05.221 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124215.log.gz 2024-09-18T01:19:05.221 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124194.log.gz 2024-09-18T01:19:05.268 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.42086.log.gz 2024-09-18T01:19:05.278 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102787.log.gz 2024-09-18T01:19:05.278 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102830.log.gz 2024-09-18T01:19:05.285 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.30580.log.gz 2024-09-18T01:19:05.289 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42265.log.gz 2024-09-18T01:19:05.335 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65542.log.gz 2024-09-18T01:19:05.340 INFO:teuthology.orchestra.run.smithi049.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.admin.108048.log.gz 2024-09-18T01:19:05.382 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.128711.log.gz 2024-09-18T01:19:05.405 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123292.log.gz 2024-09-18T01:19:05.485 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37262.log.gz 2024-09-18T01:19:05.486 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.109176.log.gz 2024-09-18T01:19:05.492 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40967.log.gz 2024-09-18T01:19:05.505 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.98998.log.gz 2024-09-18T01:19:05.516 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.admin.60939.log.gz 2024-09-18T01:19:05.542 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.75650.log.gz 2024-09-18T01:19:05.587 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104218.log.gz 2024-09-18T01:19:05.601 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.128713.log.gz 2024-09-18T01:19:05.608 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.118341.log.gz 2024-09-18T01:19:05.617 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136821.log.gz 2024-09-18T01:19:05.625 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135940.log.gz 2024-09-18T01:19:05.728 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111476.log.gz 2024-09-18T01:19:05.741 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.75522.log.gz 2024-09-18T01:19:05.753 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62878.log.gz 2024-09-18T01:19:05.753 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43698.log.gz 2024-09-18T01:19:05.807 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.118314.log.gz 2024-09-18T01:19:05.818 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.130952.log.gz 2024-09-18T01:19:05.896 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102278.log.gz 2024-09-18T01:19:05.898 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111161.log.gz 2024-09-18T01:19:05.901 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96955.log.gz 2024-09-18T01:19:05.906 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.84035.log.gz 2024-09-18T01:19:05.921 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.107793.log.gz 2024-09-18T01:19:05.981 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126736.log.gz 2024-09-18T01:19:05.994 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.51931.log.gz 2024-09-18T01:19:05.996 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.53618.log.gz 2024-09-18T01:19:06.049 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74438.log.gz 2024-09-18T01:19:06.053 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.74278.log.gz 2024-09-18T01:19:06.067 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.91697.log.gz 2024-09-18T01:19:06.069 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.84935.log.gz 2024-09-18T01:19:06.071 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.139086.log.gz 2024-09-18T01:19:06.073 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.93210.log.gz 2024-09-18T01:19:06.077 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77414.log.gz 2024-09-18T01:19:06.080 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.110865.log.gz 2024-09-18T01:19:06.084 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.75374.log.gz 2024-09-18T01:19:06.094 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101877.log.gz 2024-09-18T01:19:06.097 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111749.log.gz 2024-09-18T01:19:06.098 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62650.log.gz 2024-09-18T01:19:06.121 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.76320.log.gz 2024-09-18T01:19:06.126 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112377.log.gz 2024-09-18T01:19:06.129 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.121635.log.gz 2024-09-18T01:19:06.133 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114760.log.gz 2024-09-18T01:19:06.194 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33480.log.gz 2024-09-18T01:19:06.198 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33206.log.gz 2024-09-18T01:19:06.204 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88372.log.gz 2024-09-18T01:19:06.207 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88582.log.gz 2024-09-18T01:19:06.213 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90642.log.gz 2024-09-18T01:19:06.213 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.129695.log.gz 2024-09-18T01:19:06.217 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.87980.log.gz 2024-09-18T01:19:06.221 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86657.log.gz 2024-09-18T01:19:06.237 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70253.log.gz 2024-09-18T01:19:06.239 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107504.log.gz 2024-09-18T01:19:06.293 INFO:teuthology.orchestra.run.smithi049.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132269.log.gz 2024-09-18T01:19:06.297 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.134140.log.gz 2024-09-18T01:19:06.300 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136611.log.gz 2024-09-18T01:19:06.362 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.109850.log.gz 2024-09-18T01:19:06.368 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35159.log.gz 2024-09-18T01:19:06.372 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57265.log.gz 2024-09-18T01:19:06.373 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59624.log.gz 2024-09-18T01:19:06.376 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.30476.log.gz 2024-09-18T01:19:06.378 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33520.log.gz 2024-09-18T01:19:06.381 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster1-client.admin.42002.log.gz 2024-09-18T01:19:06.381 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34783.log.gz 2024-09-18T01:19:06.385 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.98307.log.gz 2024-09-18T01:19:06.388 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.31086.log.gz 2024-09-18T01:19:06.399 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59666.log.gz 2024-09-18T01:19:06.401 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55606.log.gz 2024-09-18T01:19:06.409 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69706.log.gz 2024-09-18T01:19:06.412 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58871.log.gz 2024-09-18T01:19:06.421 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32112.log.gz 2024-09-18T01:19:06.423 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.49541.log.gz 2024-09-18T01:19:06.433 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.107907.log.gz 2024-09-18T01:19:06.434 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77046.log.gz 2024-09-18T01:19:06.435 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111751.log.gz 2024-09-18T01:19:06.449 INFO:teuthology.orchestra.run.smithi049.stderr: 68.0% -- replaced with /var/log/ceph/cluster1-client.mirror.108115.log.gz 2024-09-18T01:19:06.459 INFO:teuthology.orchestra.run.smithi049.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.64801.log.gz 2024-09-18T01:19:06.464 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46094.log.gz 2024-09-18T01:19:06.468 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101183.log.gz 2024-09-18T01:19:06.487 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.116589.log.gz 2024-09-18T01:19:06.496 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136822.log.gz 2024-09-18T01:19:06.511 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135135.log.gz 2024-09-18T01:19:06.518 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.139358.log.gz 2024-09-18T01:19:06.528 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster1-client.admin.131134.log.gz 2024-09-18T01:19:06.530 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88705.log.gz 2024-09-18T01:19:06.531 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.139079.log.gz 2024-09-18T01:19:06.554 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34214.log.gz 2024-09-18T01:19:06.559 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78546.log.gz 2024-09-18T01:19:06.562 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.86479.log.gz 2024-09-18T01:19:06.569 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.61441.log.gz 2024-09-18T01:19:06.579 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70334.log.gz 2024-09-18T01:19:06.581 INFO:teuthology.orchestra.run.smithi049.stderr: 90.9% -- replaced with /var/log/ceph/cluster1-client.admin.74506.log.gz 2024-09-18T01:19:06.584 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66399.log.gz 2024-09-18T01:19:06.588 INFO:teuthology.orchestra.run.smithi049.stderr: 31.7% -- replaced with /var/log/ceph/cluster1-client.admin.63948.log.gz 2024-09-18T01:19:06.588 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57242.log.gz 2024-09-18T01:19:06.593 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56552.log.gz 2024-09-18T01:19:06.596 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56554.log.gz 2024-09-18T01:19:06.602 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46997.log.gz 2024-09-18T01:19:06.612 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46894.log.gz 2024-09-18T01:19:06.614 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48302.log.gz 2024-09-18T01:19:06.619 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69075.log.gz 2024-09-18T01:19:06.658 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34740.log.gz 2024-09-18T01:19:06.674 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86194.log.gz 2024-09-18T01:19:06.675 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88085.log.gz 2024-09-18T01:19:06.687 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88875.log.gz 2024-09-18T01:19:06.689 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.138561.log.gz 2024-09-18T01:19:06.690 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.57267.log.gz 2024-09-18T01:19:06.693 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46053.log.gz 2024-09-18T01:19:06.698 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34424.log.gz 2024-09-18T01:19:06.701 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92813.log.gz 2024-09-18T01:19:06.703 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.40417.log.gz 2024-09-18T01:19:06.708 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42574.log.gz 2024-09-18T01:19:06.712 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.104007.log.gz 2024-09-18T01:19:06.713 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.74857.log.gz 2024-09-18T01:19:06.717 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70082.log.gz 2024-09-18T01:19:06.721 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.61705.log.gz 2024-09-18T01:19:06.739 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44054.log.gz 2024-09-18T01:19:06.743 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101855.log.gz 2024-09-18T01:19:06.745 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103292.log.gz 2024-09-18T01:19:06.748 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44724.log.gz 2024-09-18T01:19:06.830 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.30941.log.gz 2024-09-18T01:19:06.838 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.132340.log.gz 2024-09-18T01:19:06.840 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122014.log.gz 2024-09-18T01:19:06.845 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.98177.log.gz 2024-09-18T01:19:06.848 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80908.log.gz 2024-09-18T01:19:06.850 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91402.log.gz 2024-09-18T01:19:06.854 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.39252.log.gz 2024-09-18T01:19:06.856 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90079.log.gz 2024-09-18T01:19:06.857 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90263.log.gz 2024-09-18T01:19:06.859 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.30505.log.gz 2024-09-18T01:19:06.889 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76791.log.gz 2024-09-18T01:19:06.899 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69496.log.gz 2024-09-18T01:19:06.901 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69788.log.gz 2024-09-18T01:19:06.930 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.137873.log.gz 2024-09-18T01:19:06.933 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110990.log.gz 2024-09-18T01:19:06.939 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122747.log.gz 2024-09-18T01:19:06.998 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126654.log.gz 2024-09-18T01:19:07.007 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137455.log.gz 2024-09-18T01:19:07.057 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78463.log.gz 2024-09-18T01:19:07.083 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104617.log.gz 2024-09-18T01:19:07.108 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99498.log.gz 2024-09-18T01:19:07.116 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136820.log.gz 2024-09-18T01:19:07.125 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.105906.log.gz 2024-09-18T01:19:07.135 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56764.log.gz 2024-09-18T01:19:07.152 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.55924.log.gz 2024-09-18T01:19:07.159 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.72495.log.gz 2024-09-18T01:19:07.168 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.127784.log.gz 2024-09-18T01:19:07.277 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster1-client.admin.30976.log.gz 2024-09-18T01:19:07.289 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.56217.log.gz 2024-09-18T01:19:07.297 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.33060.log.gz 2024-09-18T01:19:07.300 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102575.log.gz 2024-09-18T01:19:07.301 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91764.log.gz 2024-09-18T01:19:07.302 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91908.log.gz 2024-09-18T01:19:07.305 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82253.log.gz 2024-09-18T01:19:07.306 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87314.log.gz 2024-09-18T01:19:07.310 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88180.log.gz 2024-09-18T01:19:07.312 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45210.log.gz 2024-09-18T01:19:07.315 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45799.log.gz 2024-09-18T01:19:07.319 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47462.log.gz 2024-09-18T01:19:07.319 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47528.log.gz 2024-09-18T01:19:07.324 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71265.log.gz 2024-09-18T01:19:07.328 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65971.log.gz 2024-09-18T01:19:07.333 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.117623.log.gz 2024-09-18T01:19:07.333 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124426.log.gz 2024-09-18T01:19:07.369 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35200.log.gz 2024-09-18T01:19:07.370 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89716.log.gz 2024-09-18T01:19:07.372 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.98774.log.gz 2024-09-18T01:19:07.374 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77204.log.gz 2024-09-18T01:19:07.379 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.37973.log.gz 2024-09-18T01:19:12.016 INFO:teuthology.orchestra.run.smithi028.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2024-09-18T01:19:18.390 INFO:teuthology.orchestra.run.smithi049.stderr: 68.0% -- replaced with /var/log/ceph/cluster1-client.mirror.63106.log.gz 2024-09-18T01:19:18.390 INFO:teuthology.orchestra.run.smithi049.stderr: 67.3% -- replaced with /var/log/ceph/cluster1-client.mirror.130218.log.gz 2024-09-18T01:19:18.390 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.86238.log.gz 2024-09-18T01:19:18.391 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.60833.log.gz 2024-09-18T01:19:18.391 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112273.log.gz 2024-09-18T01:19:18.391 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127094.log.gz 2024-09-18T01:19:18.391 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66484.log.gz 2024-09-18T01:19:18.391 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.85478.log.gz 2024-09-18T01:19:18.391 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.138666.log.gz 2024-09-18T01:19:18.391 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.133667.log.gz 2024-09-18T01:19:18.391 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.114128.log.gz 2024-09-18T01:19:18.391 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78755.log.gz 2024-09-18T01:19:18.391 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.100638.log.gz 2024-09-18T01:19:18.391 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66967.log.gz 2024-09-18T01:19:18.391 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66234.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.68018.log.gz 2024-09-18T01:19:18.392 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.392 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.36949.log.gz 2024-09-18T01:19:18.392 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114000.log.gz 2024-09-18T01:19:18.392 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99925.log.gz 2024-09-18T01:19:18.392 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.84451.log.gz 2024-09-18T01:19:18.392 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88748.log.gz 2024-09-18T01:19:18.392 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.119584.log.gz 2024-09-18T01:19:18.392 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114590.log.gz 2024-09-18T01:19:18.392 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58030.log.gz 2024-09-18T01:19:18.392 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124845.log.gz 2024-09-18T01:19:18.392 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34319.log.gz 2024-09-18T01:19:18.392 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100385.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134141.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.98527.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37157.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.72128.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78335.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 67.3% -- replaced with /var/log/ceph/cluster1-client.mirror.108343.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58154.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135130.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93509.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55014.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137664.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114338.log.gz 2024-09-18T01:19:18.393 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42048.log.gz 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.123000.log.gz 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.69999.log.gz 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster1-client.admin.98798.log.gz 89.9% 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.46472.log.gz 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70929.log.gz 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.50111.log.gz 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: 58.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29925.log.gz 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41154.log.gz 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103461.log.gz 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103462.log.gz 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31355.log.gz 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36253.log.gz 2024-09-18T01:19:18.394 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46850.log.gz 2024-09-18T01:19:18.395 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55328.log.gz 2024-09-18T01:19:18.395 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33313.log.gz 2024-09-18T01:19:18.395 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.63576.log.gz 2024-09-18T01:19:18.395 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136613.log.gz 2024-09-18T01:19:18.395 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.73909.log.gz 2024-09-18T01:19:18.395 INFO:teuthology.orchestra.run.smithi049.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.63771.log.gz 2024-09-18T01:19:18.395 INFO:teuthology.orchestra.run.smithi049.stderr: 20.5% -- replaced with /var/log/ceph/cluster1-client.admin.121360.log.gz 2024-09-18T01:19:18.395 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52601.log.gz 2024-09-18T01:19:18.395 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.97137.log.gz 2024-09-18T01:19:18.395 INFO:teuthology.orchestra.run.smithi049.stderr: 18.2% -- replaced with /var/log/ceph/cluster2-client.admin.72078.log.gz 2024-09-18T01:19:18.395 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43574.log.gz 2024-09-18T01:19:18.395 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100346.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36213.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69766.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134417.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69789.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.132264.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.53175.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.123564.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59983.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.120661.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92454.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65885.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43674.log.gz 2024-09-18T01:19:18.396 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35475.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54250.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57348.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100806.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100682.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78000.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111162.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101645.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67390.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.36359.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63407.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46196.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.107934.log.gz 2024-09-18T01:19:18.397 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81665.log.gz 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58660.log.gz 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44538.log.gz 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31601.log.gz 89.9% 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.123354.log.gz 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90705.log.gz 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100553.log.gz 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% 89.9% -- replaced with /var/log/ceph/cluster2-client.admin.61041.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.138082.log.gz 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.36505.log.gz 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75698.log.gz 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88179.log.gz 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93000.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.70294.log.gz 2024-09-18T01:19:18.398 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124343.log.gz 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.95237.log.gz 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.120115.log.gz 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78839.log.gz 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58532.log.gz 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.71694.log.gz 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 18.7% -- replaced with /var/log/ceph/cluster1-client.admin.121134.log.gz 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59559.log.gz 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78673.log.gz 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster1-client.admin.119757.log.gz 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55879.log.gz 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.109316.log.gz 2024-09-18T01:19:18.399 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132725.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.131047.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37473.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81581.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.114064.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68291.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47571.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.109470.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34005.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124637.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115979.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.75760.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122955.log.gz 2024-09-18T01:19:18.400 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.69412.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.85873.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55270.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136192.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124681.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114421.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.71840.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77497.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70754.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123167.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136740.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.43357.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.139756.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.119409.log.gz 2024-09-18T01:19:18.401 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.117021.log.gz 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56088.log.gz 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112859.log.gz 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.95175.log.gz 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136191.log.gz 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster2-client.admin.140281.log.gz 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34320.log.gz 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124342.log.gz 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.72933.log.gz 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68185.log.gz 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134500.log.gz 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.90346.log.gz 2024-09-18T01:19:18.402 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113744.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.75240.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster1-client.admin.120548.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 67.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51797.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29959.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.94080.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 18.2% -- replaced with /var/log/ceph/cluster2-client.admin.95685.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122323.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.97894.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104682.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.86727.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114801.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36842.log.gz 2024-09-18T01:19:18.403 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102450.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65625.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68524.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.108887.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126527.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131556.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.129472.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.64352.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127158.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.95695.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69287.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113788.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138191.log.gz 2024-09-18T01:19:18.404 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.131085.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46726.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81706.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.98573.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.44537.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43884.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.99159.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.107128.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84760.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31970.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.74479.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.113221.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37995.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92873.log.gz 2024-09-18T01:19:18.405 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92456.log.gz 2024-09-18T01:19:18.406 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.96654.log.gz 2024-09-18T01:19:18.406 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91485.log.gz 2024-09-18T01:19:18.406 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55837.log.gz 2024-09-18T01:19:18.406 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58721.log.gz 2024-09-18T01:19:18.406 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54849.log.gz 2024-09-18T01:19:18.406 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45883.log.gz 2024-09-18T01:19:18.406 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65082.log.gz 2024-09-18T01:19:18.406 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33374.log.gz 2024-09-18T01:19:18.406 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.117972.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68440.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.100848.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.109675.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32998.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48260.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126317.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56047.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.33772.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51414.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.40558.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.31603.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.76017.log.gz 2024-09-18T01:19:18.407 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster1-client.admin.109111.log.gz 2024-09-18T01:19:18.408 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.64804.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.115758.log.gz 2024-09-18T01:19:18.408 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.408 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91552.log.gz 2024-09-18T01:19:18.408 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123691.log.gz 2024-09-18T01:19:18.408 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70907.log.gz 2024-09-18T01:19:18.408 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133787.log.gz 2024-09-18T01:19:18.408 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74760.log.gz 2024-09-18T01:19:18.408 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61656.log.gz 90.0% 2024-09-18T01:19:18.408 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.121541.log.gz 2024-09-18T01:19:18.408 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132079.log.gz 2024-09-18T01:19:18.408 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136405.log.gz 2024-09-18T01:19:18.408 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54763.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47631.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132779.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.108476.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138357.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133237.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34572.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122745.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.113010.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52538.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65947.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.120584.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.30777.log.gz 2024-09-18T01:19:18.409 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42811.log.gz 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.119847.log.gz 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45420.log.gz 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.84262.log.gz 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.31488.log.gz 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.94804.log.gz 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.63612.log.gz 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.50463.log.gz 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.130607.log.gz 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.86909.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.37263.log.gz 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32579.log.gz 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37555.log.gz 2024-09-18T01:19:18.410 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121500.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58451.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56134.log.gz 89.9% 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.58533.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67556.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47038.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55480.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.75349.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47780.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53152.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 66.6% -- replaced with /var/log/ceph/cluster1-client.mirror.107773.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.116658.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136530.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.61387.log.gz 2024-09-18T01:19:18.411 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.31511.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.131190.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135529.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122324.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.127553.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.31858.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.115682.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70396.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121415.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133069.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34888.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46306.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82402.log.gz 2024-09-18T01:19:18.412 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.64634.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.90556.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.116778.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89714.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54848.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.69035.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133070.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132500.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.84446.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135554.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90283.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66676.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44683.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.64329.log.gz 2024-09-18T01:19:18.413 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% 90.7% -- replaced with /var/log/ceph/cluster1-client.admin.63527.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.119383.log.gz 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99457.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.59921.log.gz 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.63504.log.gz 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65455.log.gz 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.128171.log.gz 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85645.log.gz 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79774.log.gz 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37367.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.69705.log.gz 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101731.log.gz 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.86968.log.gz 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67807.log.gz 2024-09-18T01:19:18.414 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136615.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111101.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.96795.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.85847.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81877.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.139428.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.118393.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45040.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88816.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 66.9% -- replaced with /var/log/ceph/cluster1-client.admin.29618.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.123795.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.113281.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111205.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50570.log.gz 2024-09-18T01:19:18.415 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.140213.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126210.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.119222.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.105442.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101983.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99160.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130177.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.97224.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.99541.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.60502.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 18.2% -- replaced with /var/log/ceph/cluster1-client.admin.97847.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.40127.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.64482.log.gz 2024-09-18T01:19:18.416 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115897.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.46938.log.gz 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136193.log.gz 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.38589.log.gz 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89508.log.gz 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70209.log.gz 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.64198.log.gz 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.83607.log.gz 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37052.log.gz 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59161.log.gz 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.42980.log.gz 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.117589.log.gz 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.117596.log.gz 2024-09-18T01:19:18.417 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35836.log.gz 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93292.log.gz 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67348.log.gz 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126781.log.gz 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.49669.log.gz 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69288.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.136407.log.gz 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42844.log.gz 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100973.log.gz 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46827.log.gz 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90076.log.gz 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111686.log.gz 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112482.log.gz 2024-09-18T01:19:18.418 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.112166.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77243.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77624.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.72496.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135559.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67765.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66945.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47206.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.120479.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.97446.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136110.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.135342.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.128165.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85075.log.gz 2024-09-18T01:19:18.419 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.52303.log.gz 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.77289.log.gz 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111266.log.gz 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137453.log.gz 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.admin.106513.log.gz 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43993.log.gz 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.36085.log.gz 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32745.log.gz 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125664.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.89422.log.gz 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68737.log.gz 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79387.log.gz 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36462.log.gz 2024-09-18T01:19:18.420 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81414.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.33730.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89297.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.114215.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.mirror.87560.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89656.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.98352.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.38585.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103628.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87918.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.110447.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104343.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43784.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.114970.log.gz 2024-09-18T01:19:18.421 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137580.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.117704.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.30869.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.129265.log.gz 90.0% 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.35098.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.108135.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65667.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62422.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37516.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.30227.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58239.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.38799.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68017.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114843.log.gz 2024-09-18T01:19:18.422 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80740.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137032.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40283.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.102743.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138002.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.116215.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59857.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58089.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.92666.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.85391.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.109396.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.75064.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125789.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.130694.log.gz 2024-09-18T01:19:18.423 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133029.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90769.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46785.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136780.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103375.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.74050.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.30752.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 55.8% -- replaced with /var/log/ceph/cluster1-client.mirror.29635.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.36423.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36360.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124679.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65414.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.30184.log.gz 2024-09-18T01:19:18.424 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89169.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59455.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103544.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.87759.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134297.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68865.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99245.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.75880.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103083.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 18.8% -- replaced with /var/log/ceph/cluster2-client.admin.39695.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 19.4% -- replaced with /var/log/ceph/cluster1-client.mirror.135900.log.gz -- replaced with /var/log/ceph/cluster2-client.admin.83763.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111810.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.91359.log.gz 2024-09-18T01:19:18.425 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.56679.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.129207.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.120422.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49792.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65843.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.114440.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68101.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115898.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 67.3% -- replaced with /var/log/ceph/cluster1-client.mirror.62992.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80279.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.126780.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65587.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136531.log.gz 2024-09-18T01:19:18.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137452.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111328.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137874.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87420.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.91616.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124680.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45211.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.109765.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111957.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.131428.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66099.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.87072.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68313.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77833.log.gz 2024-09-18T01:19:18.427 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112649.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82168.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.116218.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104301.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.110355.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.62495.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88706.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.75451.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133742.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123588.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.127242.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster2-client.admin.29823.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.140459.log.gz 2024-09-18T01:19:18.428 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40168.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.51247.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127286.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.113789.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.105223.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126863.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.32849.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48471.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57032.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74324.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.109491.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99638.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.33627.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.63267.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34049.log.gz 2024-09-18T01:19:18.429 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.117315.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.81205.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.32954.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.50004.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54319.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113009.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.123924.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.114277.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122240.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64684.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.88915.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.61353.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.126947.log.gz 2024-09-18T01:19:18.430 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.71293.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132282.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80951.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47840.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92394.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34215.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135941.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56845.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68019.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55268.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.73822.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68926.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.87225.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112237.log.gz 2024-09-18T01:19:18.431 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67895.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35580.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.72578.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.32268.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102043.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65714.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47359.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.98105.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89234.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134296.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41492.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56888.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.139759.log.gz 2024-09-18T01:19:18.432 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135090.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101143.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.114632.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.52401.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster1-client.admin.42696.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78295.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138004.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35684.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77498.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54486.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.68652.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66014.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.64728.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.124470.log.gz 2024-09-18T01:19:18.433 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.104468.log.gz 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111034.log.gz 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80448.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.36673.log.gz 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34614.log.gz 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.126505.log.gz 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54807.log.gz 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32514.log.gz 90.0% 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.33375.log.gz 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.107706.log.gz 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: 90.7% -- replaced with /var/log/ceph/cluster1-client.admin.96909.log.gz 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103629.log.gz 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.109242.log.gz 2024-09-18T01:19:18.434 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.75309.log.gz 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122451.log.gz 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137637.log.gz 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102619.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.33419.log.gz 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46302.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.79776.log.gz 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.120789.log.gz 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77916.log.gz 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100133.log.gz 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.39985.log.gz 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.139232.log.gz 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65328.log.gz 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67597.log.gz 2024-09-18T01:19:18.435 INFO:teuthology.orchestra.run.smithi049.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.74190.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.108583.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103208.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65715.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46052.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112588.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138401.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81076.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 67.3% -- replaced with /var/log/ceph/cluster1-client.mirror.129762.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80237.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.60834.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56006.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.116787.log.gz 2024-09-18T01:19:18.436 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.71583.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.87562.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89237.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115223.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.117456.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.61773.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.95236.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80573.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 65.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126445.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.96748.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.76436.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103883.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.91404.log.gz 2024-09-18T01:19:18.437 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55434.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112948.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82423.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.42422.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.130580.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.30389.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.108754.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103838.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99091.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.42549.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47906.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.82782.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125520.log.gz 2024-09-18T01:19:18.438 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43741.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47486.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.86776.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.120596.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.116522.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 18.2% -- replaced with /var/log/ceph/cluster1-client.admin.63794.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125393.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110586.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115639.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.108913.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster2-client.admin.118438.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74780.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.94518.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129876.log.gz 2024-09-18T01:19:18.439 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100471.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134308.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.116701.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102319.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110950.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.50056.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.134207.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88396.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.100134.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131531.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92663.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118725.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89632.log.gz 2024-09-18T01:19:18.440 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93807.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.54402.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41251.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.112732.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37135.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.121258.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92393.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135050.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.127264.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67599.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91614.log.gz 67.3% -- replaced with /var/log/ceph/cluster1-client.mirror.74345.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.31180.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.117018.log.gz 2024-09-18T01:19:18.441 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.97740.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.84561.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.128508.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69874.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35518.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104006.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85303.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93023.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123502.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78630.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.98842.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.91841.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.127988.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59501.log.gz 2024-09-18T01:19:18.442 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.32422.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42918.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33059.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122722.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66377.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.98484.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46154.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.64502.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32681.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67809.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136321.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.139467.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89592.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster1-client.admin.76460.log.gz 2024-09-18T01:19:18.443 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.132385.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119637.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.94033.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122242.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43908.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.70462.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57772.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 54.2% -- replaced with /var/log/ceph/cluster1-client.mirror.29737.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35055.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.100934.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40926.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111599.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56194.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48515.log.gz 2024-09-18T01:19:18.444 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133111.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125685.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.64969.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.41933.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69160.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.105471.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67138.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.104849.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91844.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78629.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131913.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46724.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80867.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76729.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.126360.log.gz 2024-09-18T01:19:18.445 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56343.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86967.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102234.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.40786.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76263.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.94806.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.71989.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.93083.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43994.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.124614.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.130674.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.75895.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.38257.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.105752.log.gz 2024-09-18T01:19:18.446 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.31901.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102065.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111682.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111539.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79435.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.113856.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88129.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126925.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133788.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.69579.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.71806.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.52624.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 67.8% -- replaced with /var/log/ceph/cluster1-client.mirror.108001.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52158.log.gz 2024-09-18T01:19:18.447 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.93231.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111204.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81748.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.83712.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52004.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110831.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.118953.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.140094.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.30204.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36212.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62401.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 67.5% -- replaced with /var/log/ceph/cluster1-client.mirror.40625.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31857.log.gz 2024-09-18T01:19:18.448 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133365.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67767.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58299.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.73037.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112523.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92876.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.43633.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.admin.107020.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55269.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster2-client.admin.29806.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.111851.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66631.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.38256.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster2-client.admin.95684.log.gz 2024-09-18T01:19:18.449 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.96768.log.gz 2024-09-18T01:19:18.450 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48261.log.gz 2024-09-18T01:19:18.450 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45000.log.gz 2024-09-18T01:19:18.450 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66097.log.gz 2024-09-18T01:19:18.450 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.32640.log.gz 2024-09-18T01:19:18.450 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121910.log.gz 2024-09-18T01:19:18.450 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.135513.log.gz 2024-09-18T01:19:18.451 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster2-client.admin.39397.log.gz 2024-09-18T01:19:18.451 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70293.log.gz 2024-09-18T01:19:18.451 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48158.log.gz 2024-09-18T01:19:18.451 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.71096.log.gz 2024-09-18T01:19:18.451 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77664.log.gz 2024-09-18T01:19:18.451 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80196.log.gz 2024-09-18T01:19:18.451 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99839.log.gz 2024-09-18T01:19:18.451 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136320.log.gz 2024-09-18T01:19:18.451 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.134710.log.gz 2024-09-18T01:19:18.451 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.107096.log.gz 2024-09-18T01:19:18.451 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.78464.log.gz 2024-09-18T01:19:18.451 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44200.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53327.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78219.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster1-client.admin.120374.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115433.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.127116.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.36715.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36885.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster1-client.admin.108957.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.96567.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122958.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88329.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126085.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55436.log.gz 2024-09-18T01:19:18.452 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.38674.log.gz 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.86107.log.gz 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.118180.log.gz 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78085.log.gz 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.117377.log.gz 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.73256.log.gz 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32464.log.gz 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.102023.log.gz 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32359.log.gz 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58870.log.gz 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80532.log.gz 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66968.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.81245.log.gz 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.453 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.110227.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.66842.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.139137.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67869.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39384.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.31095.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.105555.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69033.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33899.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88181.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100304.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.56089.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.40987.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.131228.log.gz 2024-09-18T01:19:18.454 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59710.log.gz 2024-09-18T01:19:18.455 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114699.log.gz 2024-09-18T01:19:18.455 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32577.log.gz 2024-09-18T01:19:18.455 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.44160.log.gz 2024-09-18T01:19:18.455 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125606.log.gz 2024-09-18T01:19:18.455 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80029.log.gz 2024-09-18T01:19:18.455 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67179.log.gz 2024-09-18T01:19:18.455 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72894.log.gz 2024-09-18T01:19:18.455 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115474.log.gz 2024-09-18T01:19:18.455 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58954.log.gz 2024-09-18T01:19:18.455 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.115854.log.gz 2024-09-18T01:19:18.455 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101404.log.gz 2024-09-18T01:19:18.456 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79301.log.gz 2024-09-18T01:19:18.456 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80407.log.gz 2024-09-18T01:19:18.456 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121970.log.gz 2024-09-18T01:19:18.456 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.76636.log.gz 2024-09-18T01:19:18.456 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77047.log.gz 2024-09-18T01:19:18.456 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133027.log.gz 2024-09-18T01:19:18.456 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111541.log.gz 2024-09-18T01:19:18.456 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89086.log.gz 2024-09-18T01:19:18.456 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.117883.log.gz 2024-09-18T01:19:18.456 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115748.log.gz 2024-09-18T01:19:18.456 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104049.log.gz 2024-09-18T01:19:18.456 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66274.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136823.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.131716.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 20.0% -- replaced with /var/log/ceph/cluster2-client.admin.61278.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.139893.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.50320.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.83897.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.116757.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79944.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134079.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.40444.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.69875.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.108457.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.31240.log.gz 2024-09-18T01:19:18.457 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115117.log.gz 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70419.log.gz 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.69833.log.gz 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54030.log.gz 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.48408.log.gz 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66057.log.gz 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62155.log.gz 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36527.log.gz 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50257.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.89444.log.gz 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.106949.log.gz 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.128102.log.gz 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.97397.log.gz 2024-09-18T01:19:18.458 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33562.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47990.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.138830.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35452.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54403.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71388.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.65301.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.62811.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79093.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.51703.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.56007.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89085.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42633.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81876.log.gz 2024-09-18T01:19:18.459 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.117457.log.gz 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134457.log.gz 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% 89.8% -- replaced with /var/log/ceph/cluster2-client.admin.62043.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.99926.log.gz 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.74837.log.gz 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104134.log.gz 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66819.log.gz 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.94517.log.gz 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-client.admin.63039.log.gz 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40356.log.gz 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113641.log.gz 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36317.log.gz 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51569.log.gz 2024-09-18T01:19:18.460 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51300.log.gz 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102406.log.gz 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136150.log.gz 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.38548.log.gz 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55982.log.gz 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79522.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.105137.log.gz 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34737.log.gz 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133155.log.gz 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46576.log.gz 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.71837.log.gz 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.71055.log.gz 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92032.log.gz 2024-09-18T01:19:18.461 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.78209.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.87247.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103417.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70544.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.47949.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.31061.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.138658.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.124009.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78757.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.120525.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.118672.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125186.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67557.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.121416.log.gz 2024-09-18T01:19:18.462 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99330.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44201.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.39390.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96593.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.63907.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113858.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.101519.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69832.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41385.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80155.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79345.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.48955.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81749.log.gz 2024-09-18T01:19:18.463 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56218.log.gz 2024-09-18T01:19:18.464 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123210.log.gz 2024-09-18T01:19:18.464 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57986.log.gz 2024-09-18T01:19:18.464 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.81496.log.gz 2024-09-18T01:19:18.464 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67935.log.gz 2024-09-18T01:19:18.464 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.31308.log.gz 2024-09-18T01:19:18.464 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45841.log.gz 2024-09-18T01:19:18.464 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.116282.log.gz 2024-09-18T01:19:18.464 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45314.log.gz 2024-09-18T01:19:18.464 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.99050.log.gz 2024-09-18T01:19:18.464 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.118927.log.gz 2024-09-18T01:19:18.464 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.117702.log.gz 2024-09-18T01:19:18.465 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.62381.log.gz 2024-09-18T01:19:18.465 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.43272.log.gz 2024-09-18T01:19:18.465 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.111956.log.gz 2024-09-18T01:19:18.465 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121190.log.gz 2024-09-18T01:19:18.465 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103754.log.gz 2024-09-18T01:19:18.465 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52575.log.gz 2024-09-18T01:19:18.465 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.140686.log.gz 2024-09-18T01:19:18.465 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.93851.log.gz 2024-09-18T01:19:18.465 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.91907.log.gz 2024-09-18T01:19:18.465 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.80068.log.gz 2024-09-18T01:19:18.465 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.122934.log.gz 2024-09-18T01:19:18.481 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.94646.log.gz 2024-09-18T01:19:18.481 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.43950.log.gz 2024-09-18T01:19:18.481 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101604.log.gz 2024-09-18T01:19:18.481 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.114908.log.gz 2024-09-18T01:19:18.481 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35474.log.gz 2024-09-18T01:19:18.481 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.93084.log.gz 2024-09-18T01:19:18.481 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55922.log.gz 2024-09-18T01:19:18.481 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.137875.log.gz 2024-09-18T01:19:18.481 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.134714.log.gz 2024-09-18T01:19:18.481 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.64636.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132874.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55564.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111390.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82084.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.133113.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132824.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125897.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33164.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37199.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.118652.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70990.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35054.log.gz 2024-09-18T01:19:18.482 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88001.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58825.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.108390.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88286.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76814.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138252.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53435.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111327.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.114337.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.61496.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.96882.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.95648.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.121845.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47062.log.gz 2024-09-18T01:19:18.483 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92370.log.gz 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135133.log.gz 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131253.log.gz 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33837.log.gz 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103294.log.gz 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89168.log.gz 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99968.log.gz 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.42114.log.gz 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42113.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.90053.log.gz 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42421.log.gz 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67471.log.gz 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55059.log.gz 2024-09-18T01:19:18.484 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35685.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137160.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43992.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.140654.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster2-client.admin.116869.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.83542.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103166.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.124761.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.36672.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46684.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123629.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% 18.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115640.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.94766.log.gz 2024-09-18T01:19:18.485 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100764.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.41274.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93379.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.74023.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.60136.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.96996.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57184.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.111389.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124259.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37641.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113071.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33417.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90347.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.71805.log.gz 2024-09-18T01:19:18.486 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.85505.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.91195.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.95563.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93442.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.134630.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.106109.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.41231.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52463.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48053.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44834.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110951.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35410.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133195.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.132648.log.gz 2024-09-18T01:19:18.487 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57097.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35727.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92243.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34676.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.126295.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.87831.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 67.1% -- replaced with /var/log/ceph/cluster1-client.mirror.96406.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77370.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43356.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.75065.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82584.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.104553.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.117268.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44832.log.gz 2024-09-18T01:19:18.488 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57394.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.135981.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39962.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.45251.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.129782.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.80236.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52437.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.87598.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112440.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.41741.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.103207.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.61679.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.140215.log.gz 2024-09-18T01:19:18.489 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123501.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.118558.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121700.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102277.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135984.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55224.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123165.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35728.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.32850.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37685.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.65372.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70712.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67262.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.129162.log.gz 2024-09-18T01:19:18.490 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136951.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90221.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36950.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54674.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78167.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43740.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.95128.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67261.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89212.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.49283.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54616.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114274.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.85572.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57602.log.gz 2024-09-18T01:19:18.491 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81119.log.gz 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.129923.log.gz 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.30618.log.gz 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55118.log.gz 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44177.log.gz 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.129585.log.gz 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70885.log.gz 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.139337.log.gz 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44747.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.65373.log.gz 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.95570.log.gz 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48470.log.gz 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133997.log.gz 2024-09-18T01:19:18.492 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32892.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32893.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80658.log.gz 89.9% 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.135361.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90852.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.65046.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37472.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.128396.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.30715.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53105.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.140606.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.98085.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115853.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131621.log.gz 2024-09-18T01:19:18.493 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.110741.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.138515.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.mirror.121004.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99840.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68864.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70629.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79217.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.63706.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70843.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113787.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79094.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71034.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93400.log.gz 2024-09-18T01:19:18.494 INFO:teuthology.orchestra.run.smithi049.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.62556.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45638.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45774.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44894.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.58575.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.110141.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.131427.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136402.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.109003.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92536.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48118.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.75997.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67601.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.31750.log.gz 2024-09-18T01:19:18.495 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134841.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74994.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 18.7% -- replaced with /var/log/ceph/cluster1-client.admin.132019.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.121912.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.40303.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138127.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.49281.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57945.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81370.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110375.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125310.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.59625.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43632.log.gz 2024-09-18T01:19:18.496 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.131883.log.gz 2024-09-18T01:19:18.497 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81244.log.gz 2024-09-18T01:19:18.497 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100679.log.gz 2024-09-18T01:19:18.497 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90557.log.gz 2024-09-18T01:19:18.497 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125731.log.gz 2024-09-18T01:19:18.497 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36570.log.gz 2024-09-18T01:19:18.497 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32996.log.gz 2024-09-18T01:19:18.497 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48950.log.gz 2024-09-18T01:19:18.497 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.83543.log.gz -- replaced with /var/log/ceph/cluster2-client.admin.49888.log.gz 2024-09-18T01:19:18.497 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.497 INFO:teuthology.orchestra.run.smithi049.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31023.log.gz 2024-09-18T01:19:18.497 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90851.log.gz 2024-09-18T01:19:18.497 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72808.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102149.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 66.8% -- replaced with /var/log/ceph/cluster1-client.mirror.118880.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.92956.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104513.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137832.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.70816.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99883.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.31179.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.120418.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.119128.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99119.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92242.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.128645.log.gz 2024-09-18T01:19:18.498 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137831.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.60328.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.114700.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138083.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.51019.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89296.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58616.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132842.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67516.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123386.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70673.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.50321.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132170.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138251.log.gz 2024-09-18T01:19:18.499 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137370.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86704.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125265.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster2-client.admin.50078.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59349.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112798.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123840.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49046.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79859.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.50420.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103503.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57392.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79816.log.gz 2024-09-18T01:19:18.500 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.109551.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.119014.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.112168.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.30932.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43417.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.100345.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.32465.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.132603.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.101646.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113184.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.103333.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66634.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.109829.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88285.log.gz 2024-09-18T01:19:18.501 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137663.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.50741.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137200.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54094.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69074.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81538.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138318.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67053.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46365.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47841.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.105306.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.99287.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.126106.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134924.log.gz 2024-09-18T01:19:18.502 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.128394.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37368.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79259.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.131596.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54908.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44895.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59415.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75203.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.33477.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58450.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 19.9% -- replaced with /var/log/ceph/cluster2-client.admin.129281.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.119269.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85095.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68501.log.gz 2024-09-18T01:19:18.503 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58785.log.gz 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66969.log.gz 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127052.log.gz 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.31448.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.62951.log.gz 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123848.log.gz 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67096.log.gz 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.108513.log.gz 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.134306.log.gz 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.122803.log.gz 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80824.log.gz 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.133406.log.gz 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101437.log.gz 2024-09-18T01:19:18.504 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.97374.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.75151.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44958.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40470.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124341.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.124425.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.32316.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88246.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132778.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.32932.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.74251.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58112.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.117388.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.admin.128710.log.gz 2024-09-18T01:19:18.505 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59581.log.gz 2024-09-18T01:19:18.506 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34822.log.gz 2024-09-18T01:19:18.506 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.132055.log.gz 2024-09-18T01:19:18.506 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.98154.log.gz 2024-09-18T01:19:18.506 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.64934.log.gz 2024-09-18T01:19:18.506 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.108621.log.gz 2024-09-18T01:19:18.506 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81118.log.gz 2024-09-18T01:19:18.506 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134078.log.gz 2024-09-18T01:19:18.506 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.100260.log.gz 2024-09-18T01:19:18.506 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.133912.log.gz 2024-09-18T01:19:18.506 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.99539.log.gz 2024-09-18T01:19:18.506 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79178.log.gz 2024-09-18T01:19:18.507 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.117698.log.gz 2024-09-18T01:19:18.507 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43781.log.gz 2024-09-18T01:19:18.507 INFO:teuthology.orchestra.run.smithi049.stderr: 19.3% -- replaced with /var/log/ceph/cluster2-client.admin.128712.log.gz 2024-09-18T01:19:18.507 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.126235.log.gz 2024-09-18T01:19:18.507 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44833.log.gz 2024-09-18T01:19:18.507 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60974.log.gz 2024-09-18T01:19:18.507 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123211.log.gz 2024-09-18T01:19:18.507 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.81918.log.gz 2024-09-18T01:19:18.507 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.51589.log.gz 2024-09-18T01:19:18.507 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.admin.129085.log.gz 2024-09-18T01:19:18.507 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134647.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113155.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.98049.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92601.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54930.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74304.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36780.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53554.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77499.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66444.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.123479.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.100010.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81539.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.121108.log.gz 2024-09-18T01:19:18.508 INFO:teuthology.orchestra.run.smithi049.stderr: 19.4% -- replaced with /var/log/ceph/cluster1-client.admin.42156.log.gz 2024-09-18T01:19:18.509 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.76896.log.gz 2024-09-18T01:19:18.509 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135899.log.gz 2024-09-18T01:19:18.509 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125519.log.gz 2024-09-18T01:19:18.509 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136403.log.gz 2024-09-18T01:19:18.509 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42654.log.gz 2024-09-18T01:19:18.509 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66315.log.gz 2024-09-18T01:19:18.509 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87876.log.gz 2024-09-18T01:19:18.509 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.127115.log.gz 2024-09-18T01:19:18.509 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33900.log.gz 2024-09-18T01:19:18.509 INFO:teuthology.orchestra.run.smithi049.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.130633.log.gz 2024-09-18T01:19:18.509 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90430.log.gz 2024-09-18T01:19:18.509 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.131715.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.102742.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.112439.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.127995.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.90714.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.125730.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.41913.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48664.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35315.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.64299.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91109.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.74137.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.76645.log.gz 2024-09-18T01:19:18.510 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.128698.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.76193.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.75796.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47421.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.115327.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102320.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.133450.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78294.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.121543.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.60658.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125896.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.105789.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130405.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85759.log.gz 2024-09-18T01:19:18.511 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.130124.log.gz 2024-09-18T01:19:18.512 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.49587.log.gz 2024-09-18T01:19:18.512 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.104092.log.gz 2024-09-18T01:19:18.512 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.76125.log.gz 2024-09-18T01:19:18.512 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81959.log.gz 2024-09-18T01:19:18.512 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66925.log.gz 2024-09-18T01:19:18.512 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.72960.log.gz 2024-09-18T01:19:18.512 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110448.log.gz 2024-09-18T01:19:18.512 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.63840.log.gz 2024-09-18T01:19:18.512 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35199.log.gz 2024-09-18T01:19:18.512 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121867.log.gz 2024-09-18T01:19:18.512 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37412.log.gz 2024-09-18T01:19:18.512 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81835.log.gz 2024-09-18T01:19:18.513 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.60762.log.gz 2024-09-18T01:19:18.513 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.94992.log.gz 2024-09-18T01:19:18.513 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.83009.log.gz 2024-09-18T01:19:18.513 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.61550.log.gz 2024-09-18T01:19:18.513 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.94945.log.gz 2024-09-18T01:19:18.513 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.106687.log.gz 2024-09-18T01:19:18.587 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.117973.log.gz 2024-09-18T01:19:18.670 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.106730.log.gz 2024-09-18T01:19:18.672 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.128979.log.gz 2024-09-18T01:19:18.688 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.125454.log.gz 2024-09-18T01:19:18.746 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.50642.log.gz 2024-09-18T01:19:18.749 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.40101.log.gz 2024-09-18T01:19:18.751 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.49399.log.gz 2024-09-18T01:19:18.758 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.84706.log.gz 2024-09-18T01:19:18.762 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82338.log.gz 2024-09-18T01:19:18.764 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.73310.log.gz 2024-09-18T01:19:18.778 INFO:teuthology.orchestra.run.smithi049.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.93875.log.gz 2024-09-18T01:19:18.784 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.49356.log.gz 2024-09-18T01:19:18.836 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.109803.log.gz 2024-09-18T01:19:18.861 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.121679.log.gz 2024-09-18T01:19:18.946 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.49409.log.gz 2024-09-18T01:19:18.958 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.128491.log.gz 2024-09-18T01:19:19.010 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41060.log.gz 2024-09-18T01:19:19.020 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.114073.log.gz 2024-09-18T01:19:19.161 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73868.log.gz 2024-09-18T01:19:19.228 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119657.log.gz 2024-09-18T01:19:19.228 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40604.log.gz 2024-09-18T01:19:19.228 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86007.log.gz 2024-09-18T01:19:19.228 INFO:teuthology.orchestra.run.smithi049.stderr: 91.0% -- replaced with /var/log/ceph/cluster1-client.admin.63153.log.gz 2024-09-18T01:19:19.237 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.128060.log.gz 2024-09-18T01:19:19.346 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.38393.log.gz 2024-09-18T01:19:19.366 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.97069.log.gz 2024-09-18T01:19:19.404 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.admin.72601.log.gz 2024-09-18T01:19:19.407 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.83008.log.gz 2024-09-18T01:19:19.426 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.119714.log.gz 2024-09-18T01:19:19.465 INFO:teuthology.orchestra.run.smithi049.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.120699.log.gz 2024-09-18T01:19:19.624 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.139574.log.gz 2024-09-18T01:19:19.763 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130767.log.gz 2024-09-18T01:19:19.765 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.73708.log.gz 2024-09-18T01:19:19.770 INFO:teuthology.orchestra.run.smithi049.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.140304.log.gz 2024-09-18T01:19:19.772 INFO:teuthology.orchestra.run.smithi049.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.admin.128187.log.gz 2024-09-18T01:19:19.775 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.84172.log.gz 2024-09-18T01:19:19.797 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.91696.log.gz 2024-09-18T01:19:19.936 INFO:teuthology.orchestra.run.smithi049.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-client.admin.106378.log.gz 2024-09-18T01:19:19.957 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.61468.log.gz 2024-09-18T01:19:19.958 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.86084.log.gz 2024-09-18T01:19:19.963 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.128322.log.gz 2024-09-18T01:19:19.971 INFO:teuthology.orchestra.run.smithi049.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.94721.log.gz 2024-09-18T01:19:19.981 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.102574.log.gz 2024-09-18T01:19:19.985 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82190.log.gz 2024-09-18T01:19:20.020 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.117053.log.gz 2024-09-18T01:19:20.148 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.81079.log.gz 2024-09-18T01:19:20.150 INFO:teuthology.orchestra.run.smithi049.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-client.admin.50490.log.gz 2024-09-18T01:19:20.153 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.60920.log.gz 2024-09-18T01:19:20.155 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.61915.log.gz 2024-09-18T01:19:20.350 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.61542.log.gz 2024-09-18T01:19:20.351 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.117138.log.gz 2024-09-18T01:19:20.358 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47272.log.gz 2024-09-18T01:19:20.363 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.118088.log.gz 2024-09-18T01:19:20.476 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.101353.log.gz 2024-09-18T01:19:20.542 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.admin.95393.log.gz 2024-09-18T01:19:20.549 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.138887.log.gz 2024-09-18T01:19:20.560 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.118092.log.gz 2024-09-18T01:19:20.564 INFO:teuthology.orchestra.run.smithi049.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.108642.log.gz 2024-09-18T01:19:20.598 INFO:teuthology.orchestra.run.smithi049.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.mirror.130916.log.gz 2024-09-18T01:19:20.735 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.106517.log.gz 2024-09-18T01:19:20.735 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.106756.log.gz 2024-09-18T01:19:20.738 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.84501.log.gz 2024-09-18T01:19:20.748 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.39274.log.gz 2024-09-18T01:19:20.773 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66400.log.gz 2024-09-18T01:19:20.807 INFO:teuthology.orchestra.run.smithi049.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.64859.log.gz 2024-09-18T01:19:20.950 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118859.log.gz 2024-09-18T01:19:20.953 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118973.log.gz 2024-09-18T01:19:20.996 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46263.log.gz 2024-09-18T01:19:21.155 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.128926.log.gz 2024-09-18T01:19:21.286 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.admin.50059.log.gz 2024-09-18T01:19:21.288 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.117153.log.gz 2024-09-18T01:19:21.363 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.139539.log.gz 2024-09-18T01:19:21.372 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.139561.log.gz 2024-09-18T01:19:21.388 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37202.log.gz 2024-09-18T01:19:21.389 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.96339.log.gz 2024-09-18T01:19:21.391 INFO:teuthology.orchestra.run.smithi049.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.admin.38928.log.gz 2024-09-18T01:19:21.578 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108436.log.gz 2024-09-18T01:19:21.582 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119543.log.gz 2024-09-18T01:19:21.994 INFO:teuthology.orchestra.run.smithi049.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.mirror.41876.log.gz 2024-09-18T01:19:21.999 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74210.log.gz 2024-09-18T01:19:21.999 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.38937.log.gz 2024-09-18T01:19:22.000 INFO:teuthology.orchestra.run.smithi049.stderr: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.29652.log.gz 2024-09-18T01:19:22.006 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.96453.log.gz 2024-09-18T01:19:22.038 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46220.log.gz 2024-09-18T01:19:22.121 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113366.log.gz 2024-09-18T01:19:22.175 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134458.log.gz 2024-09-18T01:19:22.203 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.117134.log.gz 2024-09-18T01:19:22.322 INFO:teuthology.orchestra.run.smithi049.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.140303.log.gz 2024-09-18T01:19:22.412 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77789.log.gz 2024-09-18T01:19:22.420 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93992.log.gz 2024-09-18T01:19:22.424 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.84122.log.gz 2024-09-18T01:19:22.565 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.61039.log.gz 2024-09-18T01:19:22.718 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.132980.log.gz 2024-09-18T01:19:22.737 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51434.log.gz 2024-09-18T01:19:22.908 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.130083.log.gz 2024-09-18T01:19:23.319 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51776.log.gz 2024-09-18T01:19:23.347 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.72591.log.gz 2024-09-18T01:19:23.533 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67515.log.gz 2024-09-18T01:19:23.538 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.100514.log.gz 2024-09-18T01:19:23.542 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.140040.log.gz 2024-09-18T01:19:23.762 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.110544.log.gz 2024-09-18T01:19:23.907 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.114906.log.gz 2024-09-18T01:19:24.064 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.128314.log.gz 2024-09-18T01:19:24.074 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.110425.log.gz 2024-09-18T01:19:24.209 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.118080.log.gz 2024-09-18T01:19:24.376 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.72228.log.gz 2024-09-18T01:19:24.515 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107980.log.gz 2024-09-18T01:19:24.575 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82126.log.gz 2024-09-18T01:19:24.577 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85551.log.gz 2024-09-18T01:19:24.579 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.62583.log.gz 2024-09-18T01:19:24.586 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.130311.log.gz 2024-09-18T01:19:24.753 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.129741.log.gz 2024-09-18T01:19:24.776 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.105735.log.gz 2024-09-18T01:19:24.778 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.96225.log.gz 2024-09-18T01:19:24.786 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.50793.log.gz 2024-09-18T01:19:25.277 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.129530.log.gz 2024-09-18T01:19:25.322 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74666.log.gz 2024-09-18T01:19:25.394 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.116731.log.gz 2024-09-18T01:19:26.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.134498.log.gz 2024-09-18T01:19:26.071 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.127994.log.gz 2024-09-18T01:19:26.083 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74552.log.gz 2024-09-18T01:19:26.388 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41174.log.gz 2024-09-18T01:19:26.839 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.128963.log.gz 2024-09-18T01:19:27.280 INFO:teuthology.orchestra.run.smithi049.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.mirror.30167.log.gz 2024-09-18T01:19:27.305 INFO:teuthology.orchestra.run.smithi049.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.138767.log.gz 2024-09-18T01:19:27.306 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130653.log.gz 2024-09-18T01:19:27.370 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96385.log.gz 2024-09-18T01:19:27.448 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.admin.73449.log.gz 2024-09-18T01:19:27.714 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.113491.log.gz 2024-09-18T01:19:28.075 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.106742.log.gz 2024-09-18T01:19:28.078 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43247.log.gz 2024-09-18T01:19:28.304 INFO:teuthology.orchestra.run.smithi049.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.108276.log.gz 2024-09-18T01:19:28.362 INFO:teuthology.orchestra.run.smithi049.stderr: 94.1% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2024-09-18T01:19:28.418 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.129855.log.gz 2024-09-18T01:19:28.460 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.107592.log.gz 2024-09-18T01:19:28.713 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51320.log.gz 2024-09-18T01:19:28.889 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.105572.log.gz 2024-09-18T01:19:28.939 INFO:teuthology.orchestra.run.smithi049.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36738.log.gz 2024-09-18T01:19:29.696 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.61919.log.gz 2024-09-18T01:19:30.479 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.83661.log.gz 2024-09-18T01:19:30.658 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.60712.log.gz 2024-09-18T01:19:30.842 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96727.log.gz 2024-09-18T01:19:31.183 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.102874.log.gz 2024-09-18T01:19:31.187 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.48912.log.gz 2024-09-18T01:19:31.942 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.94673.log.gz 2024-09-18T01:19:32.034 INFO:teuthology.orchestra.run.smithi049.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136360.log.gz 2024-09-18T01:19:32.262 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107866.log.gz 2024-09-18T01:19:32.291 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.admin.61630.log.gz 2024-09-18T01:19:32.535 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.94485.log.gz 2024-09-18T01:19:32.546 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.107638.log.gz 2024-09-18T01:19:33.132 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84981.log.gz 2024-09-18T01:19:33.310 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.40376.log.gz 2024-09-18T01:19:33.783 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.117486.log.gz 2024-09-18T01:19:35.101 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85209.log.gz 2024-09-18T01:19:35.101 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.129969.log.gz 2024-09-18T01:19:35.101 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.38397.log.gz 2024-09-18T01:19:35.839 INFO:teuthology.orchestra.run.smithi049.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.118699.log.gz 2024-09-18T01:19:36.122 INFO:teuthology.orchestra.run.smithi049.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.admin.128312.log.gz 2024-09-18T01:19:36.126 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96841.log.gz 2024-09-18T01:19:37.165 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119087.log.gz 2024-09-18T01:19:37.487 INFO:teuthology.orchestra.run.smithi049.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.107035.log.gz 2024-09-18T01:19:38.079 INFO:teuthology.orchestra.run.smithi049.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.49723.log.gz 2024-09-18T01:19:38.088 INFO:teuthology.orchestra.run.smithi049.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.116349.log.gz 2024-09-18T01:19:41.862 INFO:teuthology.orchestra.run.smithi049.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.75524.log.gz 2024-09-18T01:19:44.787 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.130197.log.gz 2024-09-18T01:19:47.361 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63085.log.gz 2024-09-18T01:19:48.923 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96499.log.gz 2024-09-18T01:20:00.269 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62743.log.gz 2024-09-18T01:20:03.073 INFO:teuthology.orchestra.run.smithi049.stderr: 96.2% -- replaced with /var/log/ceph/cluster2-client.mirror.62857.log.gz 2024-09-18T01:20:03.405 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62971.log.gz 2024-09-18T01:20:09.299 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62629.log.gz 2024-09-18T01:20:22.952 INFO:teuthology.orchestra.run.smithi049.stderr: 96.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62515.log.gz 2024-09-18T01:27:25.422 INFO:teuthology.orchestra.run.smithi049.stderr: 96.0% -- replaced with /var/log/ceph/cluster2-client.mirror.29986.log.gz 2024-09-18T01:33:44.925 INFO:teuthology.orchestra.run.smithi028.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2024-09-18T01:33:53.715 INFO:teuthology.orchestra.run.smithi028.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2024-09-18T01:34:45.454 INFO:teuthology.orchestra.run.smithi028.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2024-09-18T01:34:45.455 INFO:teuthology.orchestra.run.smithi028.stderr: 2024-09-18T01:34:45.455 INFO:teuthology.orchestra.run.smithi028.stderr:real 15m59.110s 2024-09-18T01:34:45.455 INFO:teuthology.orchestra.run.smithi028.stderr:user 15m1.561s 2024-09-18T01:34:45.455 INFO:teuthology.orchestra.run.smithi028.stderr:sys 0m45.363s 2024-09-18T01:40:25.412 INFO:teuthology.orchestra.run.smithi049.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2024-09-18T01:43:25.493 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.39484.log.gz 2024-09-18T01:43:49.627 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.84806.log.gz 2024-09-18T01:44:23.701 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.116880.log.gz 2024-09-18T01:44:23.709 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.61089.log.gz 2024-09-18T01:44:25.290 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.95737.log.gz 2024-09-18T01:44:25.357 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.117784.log.gz 2024-09-18T01:44:29.749 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.117737.log.gz 2024-09-18T01:44:35.595 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.64361.log.gz 2024-09-18T01:44:37.073 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.63665.log.gz 2024-09-18T01:44:37.565 INFO:teuthology.orchestra.run.smithi049.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.admin.83834.log.gz 2024-09-18T01:44:43.893 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.97406.log.gz 2024-09-18T01:44:43.907 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.108679.log.gz 2024-09-18T01:44:43.909 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.63959.log.gz 2024-09-18T01:44:47.250 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.84829.log.gz 2024-09-18T01:45:00.589 INFO:teuthology.orchestra.run.smithi049.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.admin.84570.log.gz 2024-09-18T01:45:04.355 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.95971.log.gz 2024-09-18T01:45:10.376 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.87279.log.gz 2024-09-18T01:45:10.381 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.105240.log.gz 2024-09-18T01:45:10.385 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.131143.log.gz 2024-09-18T01:45:13.033 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.95696.log.gz 2024-09-18T01:45:15.135 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.83789.log.gz 2024-09-18T01:45:15.622 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.86910.log.gz 2024-09-18T01:45:25.826 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.62225.log.gz 2024-09-18T01:45:25.828 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.42165.log.gz 2024-09-18T01:45:25.830 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.39570.log.gz 2024-09-18T01:45:25.832 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.109902.log.gz 2024-09-18T01:45:25.834 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.99007.log.gz 2024-09-18T01:45:25.835 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.52356.log.gz 2024-09-18T01:45:25.836 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.61305.log.gz 2024-09-18T01:45:26.050 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.73594.log.gz 2024-09-18T01:45:26.533 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.107185.log.gz 2024-09-18T01:45:29.454 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.86736.log.gz 2024-09-18T01:45:29.979 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.73104.log.gz 2024-09-18T01:45:32.147 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.96061.log.gz 2024-09-18T01:45:32.150 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.73566.log.gz 2024-09-18T01:45:32.516 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.128835.log.gz 2024-09-18T01:45:33.934 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.84271.log.gz 2024-09-18T01:45:34.377 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.49174.log.gz 2024-09-18T01:45:34.610 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.50377.log.gz 2024-09-18T01:45:35.145 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.41417.log.gz 2024-09-18T01:45:35.859 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.75659.log.gz 2024-09-18T01:45:36.473 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.50203.log.gz 2024-09-18T01:45:45.188 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.42705.log.gz 2024-09-18T01:45:45.197 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.117732.log.gz 2024-09-18T01:45:45.203 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.52495.log.gz 2024-09-18T01:45:45.207 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.50128.log.gz 2024-09-18T01:45:48.532 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.106838.log.gz 2024-09-18T01:45:50.480 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.53627.log.gz 2024-09-18T01:45:52.781 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.52944.log.gz 2024-09-18T01:45:53.192 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.140748.log.gz 2024-09-18T01:45:53.743 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.98807.log.gz 2024-09-18T01:45:54.177 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.39994.log.gz 2024-09-18T01:45:54.253 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.74889.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.76225.log.gz 2024-09-18T01:45:54.253 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T01:45:54.253 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.50174.log.gz 2024-09-18T01:45:55.143 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.64555.log.gz 2024-09-18T01:45:59.875 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.50710.log.gz 2024-09-18T01:45:59.888 INFO:teuthology.orchestra.run.smithi049.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.admin.73116.log.gz 2024-09-18T01:45:59.890 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.83968.log.gz 2024-09-18T01:45:59.897 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.63803.log.gz 2024-09-18T01:45:59.898 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.131346.log.gz 2024-09-18T01:45:59.899 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.50902.log.gz 2024-09-18T01:46:00.336 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.139535.log.gz 2024-09-18T01:46:01.191 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.39474.log.gz 2024-09-18T01:46:01.478 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.121369.log.gz 2024-09-18T01:46:03.832 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.61689.log.gz 2024-09-18T01:46:04.572 INFO:teuthology.orchestra.run.smithi049.stderr: 95.1% -- replaced with /var/log/ceph/cluster1-client.admin.120557.log.gz 2024-09-18T01:46:08.722 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% 95.2% 94.7% -- replaced with /var/log/ceph/cluster1-client.admin.86247.log.gz 2024-09-18T01:46:08.722 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.106185.log.gz 2024-09-18T01:46:08.722 INFO:teuthology.orchestra.run.smithi049.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.97856.log.gz 2024-09-18T01:46:08.753 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.96024.log.gz 2024-09-18T01:46:08.947 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.61101.log.gz 2024-09-18T01:46:08.947 INFO:teuthology.orchestra.run.smithi049.stderr: 95.1% -- replaced with /var/log/ceph/cluster1-client.admin.119766.log.gz 2024-09-18T01:46:08.990 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.84596.log.gz 2024-09-18T01:46:08.996 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.50928.log.gz 2024-09-18T01:46:09.112 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.140350.log.gz 2024-09-18T01:46:09.791 INFO:teuthology.orchestra.run.smithi049.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.admin.83969.log.gz 2024-09-18T01:46:11.182 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.39752.log.gz 2024-09-18T01:46:11.919 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.53904.log.gz 2024-09-18T01:46:12.489 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.41283.log.gz 2024-09-18T01:46:12.989 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.75160.log.gz 2024-09-18T01:46:18.589 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.129354.log.gz 2024-09-18T01:46:18.590 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.61057.log.gz 2024-09-18T01:46:18.596 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.118402.log.gz 2024-09-18T01:46:18.596 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.50147.log.gz 2024-09-18T01:46:18.606 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.52227.log.gz 2024-09-18T01:46:19.077 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.98361.log.gz 2024-09-18T01:46:19.707 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.39737.log.gz 2024-09-18T01:46:20.676 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.39706.log.gz 2024-09-18T01:46:20.906 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.61988.log.gz 2024-09-18T01:46:20.944 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.87726.log.gz 2024-09-18T01:46:21.061 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.140710.log.gz 2024-09-18T01:46:21.571 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.139498.log.gz 2024-09-18T01:46:22.558 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.83780.log.gz 2024-09-18T01:46:22.824 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.120761.log.gz 2024-09-18T01:46:25.373 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.73038.log.gz 2024-09-18T01:46:25.380 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.72087.log.gz 2024-09-18T01:46:25.382 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.96023.log.gz 2024-09-18T01:46:25.383 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.31317.log.gz 2024-09-18T01:46:25.385 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.97673.log.gz 2024-09-18T01:46:26.819 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.117722.log.gz 2024-09-18T01:46:26.900 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.106012.log.gz 2024-09-18T01:46:27.497 INFO:teuthology.orchestra.run.smithi049.stderr: 95.4% -- replaced with /var/log/ceph/cluster1-client.admin.97535.log.gz 2024-09-18T01:46:27.497 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.131678.log.gz 2024-09-18T01:46:27.665 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.73063.log.gz 2024-09-18T01:46:29.813 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.120964.log.gz 2024-09-18T01:46:32.536 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.76469.log.gz 2024-09-18T01:46:32.593 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.129283.log.gz 2024-09-18T01:46:32.596 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.132439.log.gz 2024-09-18T01:46:32.602 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.52633.log.gz 2024-09-18T01:46:32.602 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.117991.log.gz 2024-09-18T01:46:32.786 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.73126.log.gz 2024-09-18T01:46:34.368 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.129395.log.gz 2024-09-18T01:46:36.751 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.53201.log.gz 2024-09-18T01:46:36.813 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.129310.log.gz 2024-09-18T01:46:36.878 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.41550.log.gz 2024-09-18T01:46:36.924 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.86425.log.gz 2024-09-18T01:46:36.980 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.62245.log.gz 2024-09-18T01:46:40.244 INFO:teuthology.orchestra.run.smithi049.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.admin.62013.log.gz 2024-09-18T01:46:40.253 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.131005.log.gz 2024-09-18T01:46:40.254 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.87104.log.gz 2024-09-18T01:46:40.257 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.39525.log.gz 2024-09-18T01:46:40.258 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.109707.log.gz 2024-09-18T01:46:41.067 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.107211.log.gz 2024-09-18T01:46:43.388 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.52791.log.gz 2024-09-18T01:46:43.431 INFO:teuthology.orchestra.run.smithi049.stderr: 95.1% -- replaced with /var/log/ceph/cluster1-client.admin.86116.log.gz 2024-09-18T01:46:43.518 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.41837.log.gz 2024-09-18T01:46:44.315 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.75318.log.gz 2024-09-18T01:46:44.767 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.119905.log.gz 2024-09-18T01:46:44.873 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.39791.log.gz 2024-09-18T01:46:45.711 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.140706.log.gz 2024-09-18T01:46:46.311 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.132028.log.gz 2024-09-18T01:46:46.697 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.53380.log.gz 2024-09-18T01:46:46.736 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.95738.log.gz 2024-09-18T01:46:47.492 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.129353.log.gz 2024-09-18T01:46:47.581 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.131853.log.gz 2024-09-18T01:46:47.625 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.96043.log.gz 2024-09-18T01:46:48.382 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.118455.log.gz 2024-09-18T01:46:48.786 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.117828.log.gz 2024-09-18T01:46:49.279 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.42473.log.gz 2024-09-18T01:46:49.523 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.65219.log.gz 2024-09-18T01:46:50.170 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.41704.log.gz 2024-09-18T01:46:50.893 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.118463.log.gz 2024-09-18T01:46:50.954 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.121162.log.gz 2024-09-18T01:46:51.460 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.42011.log.gz 2024-09-18T01:46:52.700 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.139776.log.gz 2024-09-18T01:46:54.037 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.31759.log.gz 2024-09-18T01:46:55.324 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.109274.log.gz 2024-09-18T01:46:56.181 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.75026.log.gz 2024-09-18T01:46:56.182 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.106907.log.gz 2024-09-18T01:46:57.279 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.42319.log.gz 2024-09-18T01:47:00.698 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.120057.log.gz 2024-09-18T01:47:01.221 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.30398.log.gz 2024-09-18T01:47:01.478 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.73193.log.gz 2024-09-18T01:47:01.529 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.39565.log.gz 2024-09-18T01:47:01.748 INFO:teuthology.orchestra.run.smithi049.stderr: 95.1% -- replaced with /var/log/ceph/cluster1-client.admin.98010.log.gz 2024-09-18T01:47:02.273 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.108545.log.gz 2024-09-18T01:47:02.274 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.86582.log.gz 2024-09-18T01:47:02.658 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.110112.log.gz 2024-09-18T01:47:02.921 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.98186.log.gz 2024-09-18T01:47:03.214 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.130876.log.gz 2024-09-18T01:47:03.544 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.30830.log.gz 2024-09-18T01:47:03.565 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.120223.log.gz 2024-09-18T01:47:04.320 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.76049.log.gz 2024-09-18T01:47:04.353 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.75849.log.gz 2024-09-18T01:47:04.463 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.63536.log.gz 2024-09-18T01:47:05.385 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.107243.log.gz 2024-09-18T01:47:05.452 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.30985.log.gz 2024-09-18T01:47:05.724 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.120383.log.gz 2024-09-18T01:47:05.799 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.64207.log.gz 2024-09-18T01:47:06.731 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.50351.log.gz 2024-09-18T01:47:07.092 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.139777.log.gz 2024-09-18T01:47:07.097 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.108812.log.gz 2024-09-18T01:47:07.812 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.30542.log.gz 2024-09-18T01:47:07.887 INFO:teuthology.orchestra.run.smithi049.stderr: 95.4% -- replaced with /var/log/ceph/cluster1-client.admin.30236.log.gz 2024-09-18T01:47:08.992 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.31561.log.gz 2024-09-18T01:47:09.269 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.65003.log.gz 2024-09-18T01:47:09.411 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.129594.log.gz 2024-09-18T01:47:10.518 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.94775.log.gz 2024-09-18T01:47:10.986 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.61289.log.gz 2024-09-18T01:47:11.148 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.94094.log.gz 2024-09-18T01:47:11.626 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.128832.log.gz 2024-09-18T01:47:11.719 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.128759.log.gz 2024-09-18T01:47:12.353 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.98536.log.gz 2024-09-18T01:47:12.509 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.131500.log.gz 2024-09-18T01:47:12.747 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.30676.log.gz 2024-09-18T01:47:13.123 INFO:teuthology.orchestra.run.smithi049.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.admin.139819.log.gz 2024-09-18T01:47:13.338 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.129481.log.gz 2024-09-18T01:47:13.738 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.75483.log.gz 2024-09-18T01:47:14.539 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.106040.log.gz 2024-09-18T01:47:14.612 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.108966.log.gz 2024-09-18T01:47:15.781 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.132223.log.gz 2024-09-18T01:47:16.053 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.109120.log.gz 2024-09-18T01:47:17.605 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.140328.log.gz 2024-09-18T01:47:18.249 INFO:teuthology.orchestra.run.smithi049.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.admin.31140.log.gz 2024-09-18T01:47:19.885 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.64763.log.gz 2024-09-18T01:47:22.934 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.106818.log.gz 2024-09-18T01:47:23.057 INFO:teuthology.orchestra.run.smithi049.stderr: 94.7% -- replaced with /var/log/ceph/cluster2-client.admin.73372.log.gz 2024-09-18T01:47:23.261 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.87474.log.gz 2024-09-18T01:47:26.121 INFO:teuthology.orchestra.run.smithi049.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.admin.109523.log.gz 2024-09-18T01:47:35.250 INFO:teuthology.orchestra.run.smithi049.stderr: 94.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.29986.log.gz 2024-09-18T01:54:17.055 INFO:teuthology.orchestra.run.smithi049.stderr: 96.9% -- replaced with /var/log/ceph/cluster1-client.mirror.30015.log.gz 2024-09-18T02:02:52.425 INFO:teuthology.orchestra.run.smithi049.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.0.30015.log.gz 2024-09-18T02:19:55.156 INFO:teuthology.orchestra.run.smithi049.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2024-09-18T02:21:10.011 INFO:teuthology.orchestra.run.smithi049.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2024-09-18T02:22:32.466 INFO:teuthology.orchestra.run.smithi049.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2024-09-18T02:22:32.487 INFO:teuthology.orchestra.run.smithi049.stderr: 2024-09-18T02:22:32.487 INFO:teuthology.orchestra.run.smithi049.stderr:real 63m46.099s 2024-09-18T02:22:32.487 INFO:teuthology.orchestra.run.smithi049.stderr:user 55m8.196s 2024-09-18T02:22:32.487 INFO:teuthology.orchestra.run.smithi049.stderr:sys 2m59.064s 2024-09-18T02:22:32.487 INFO:tasks.ceph:Archiving logs... 2024-09-18T02:22:32.488 DEBUG:teuthology.misc:Transferring archived files from smithi028:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601/remote/smithi028/log 2024-09-18T02:22:32.489 DEBUG:teuthology.orchestra.run.smithi028:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-18T02:23:09.188 DEBUG:teuthology.misc:Transferring archived files from smithi049:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601/remote/smithi049/log 2024-09-18T02:23:09.189 DEBUG:teuthology.orchestra.run.smithi049:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-18T02:25:19.424 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-18T02:25:19.435 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-18T02:25:19.435 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-18T02:25:19.438 DEBUG:teuthology.orchestra.run.smithi049:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-18T02:25:19.584 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-18T02:25:19.593 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-18T02:25:19.594 DEBUG:teuthology.orchestra.run.smithi028:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-18T02:25:19.596 DEBUG:teuthology.orchestra.run.smithi049:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-18T02:25:19.674 INFO:teuthology.orchestra.run.smithi049.stdout: remote refid st t when poll reach delay offset jitter 2024-09-18T02:25:19.674 INFO:teuthology.orchestra.run.smithi049.stdout:============================================================================== 2024-09-18T02:25:19.674 INFO:teuthology.orchestra.run.smithi049.stdout:*hv01.front.sepi 67.205.162.81 3 u 155 256 377 0.182 -1.105 0.465 2024-09-18T02:25:19.674 INFO:teuthology.orchestra.run.smithi049.stdout:+hv02.front.sepi 63.231.80.2 3 u 192 256 377 0.220 +0.127 0.917 2024-09-18T02:25:19.675 INFO:teuthology.orchestra.run.smithi049.stdout:+hv03.front.sepi 74.6.168.72 3 u 211 256 377 0.208 -4.015 0.664 2024-09-18T02:25:19.675 INFO:teuthology.orchestra.run.smithi049.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-09-18T02:25:19.694 INFO:teuthology.orchestra.run.smithi028.stdout: remote refid st t when poll reach delay offset jitter 2024-09-18T02:25:19.694 INFO:teuthology.orchestra.run.smithi028.stdout:============================================================================== 2024-09-18T02:25:19.694 INFO:teuthology.orchestra.run.smithi028.stdout:+hv01.front.sepi 67.205.162.81 3 u 66 256 377 0.234 -0.002 0.023 2024-09-18T02:25:19.694 INFO:teuthology.orchestra.run.smithi028.stdout:*hv02.front.sepi 63.231.80.2 3 u 98 256 377 0.194 +0.568 0.083 2024-09-18T02:25:19.694 INFO:teuthology.orchestra.run.smithi028.stdout:+hv03.front.sepi 74.6.168.72 3 u 80 256 377 0.175 -2.182 0.053 2024-09-18T02:25:19.694 INFO:teuthology.orchestra.run.smithi028.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-09-18T02:25:19.695 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-18T02:25:19.704 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-18T02:25:19.705 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-18T02:25:19.712 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-18T02:25:19.721 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-18T02:25:19.729 INFO:teuthology.task.internal:Duration was 10164.348619 seconds 2024-09-18T02:25:19.729 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-18T02:25:19.738 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-18T02:25:19.738 DEBUG:teuthology.orchestra.run.smithi028:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-18T02:25:19.740 DEBUG:teuthology.orchestra.run.smithi049:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-18T02:25:20.024 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-18T02:25:20.025 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi028.front.sepia.ceph.com 2024-09-18T02:25:20.025 DEBUG:teuthology.orchestra.run.smithi028:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-18T02:25:20.049 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi049.front.sepia.ceph.com 2024-09-18T02:25:20.049 DEBUG:teuthology.orchestra.run.smithi049:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-18T02:25:20.077 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-18T02:25:20.077 DEBUG:teuthology.orchestra.run.smithi028:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-18T02:25:20.092 DEBUG:teuthology.orchestra.run.smithi049:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-18T02:25:20.276 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-18T02:25:20.276 DEBUG:teuthology.orchestra.run.smithi028:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-18T02:25:20.278 DEBUG:teuthology.orchestra.run.smithi049:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-18T02:25:20.564 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-18T02:25:20.580 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-18T02:25:20.580 DEBUG:teuthology.orchestra.run.smithi028:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-18T02:25:20.636 DEBUG:teuthology.orchestra.run.smithi049:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-18T02:25:20.674 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-18T02:25:20.684 DEBUG:teuthology.orchestra.run.smithi028:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-18T02:25:20.687 DEBUG:teuthology.orchestra.run.smithi049:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-18T02:25:20.715 INFO:teuthology.orchestra.run.smithi028.stdout:kernel.core_pattern = core 2024-09-18T02:25:20.735 INFO:teuthology.orchestra.run.smithi049.stdout:kernel.core_pattern = core 2024-09-18T02:25:20.766 DEBUG:teuthology.orchestra.run.smithi028:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-18T02:25:20.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T02:25:20.792 DEBUG:teuthology.orchestra.run.smithi049:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-18T02:25:20.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T02:25:20.815 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-18T02:25:20.825 INFO:teuthology.task.internal:Transferring archived files... 2024-09-18T02:25:20.827 DEBUG:teuthology.misc:Transferring archived files from smithi028:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601/remote/smithi028 2024-09-18T02:25:20.827 DEBUG:teuthology.orchestra.run.smithi028:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-18T02:25:20.857 DEBUG:teuthology.misc:Transferring archived files from smithi049:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909601/remote/smithi049 2024-09-18T02:25:20.857 DEBUG:teuthology.orchestra.run.smithi049:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-18T02:25:20.881 INFO:teuthology.task.internal:Removing archive directory... 2024-09-18T02:25:20.881 DEBUG:teuthology.orchestra.run.smithi028:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-18T02:25:20.901 DEBUG:teuthology.orchestra.run.smithi049:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-18T02:25:20.927 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-18T02:25:20.938 INFO:teuthology.task.internal:Not uploading archives. 2024-09-18T02:25:20.938 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-18T02:25:20.947 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-18T02:25:20.947 DEBUG:teuthology.orchestra.run.smithi028:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-18T02:25:20.950 DEBUG:teuthology.orchestra.run.smithi049:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-18T02:25:20.953 INFO:teuthology.orchestra.run.smithi028.stdout: 658641 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 18 02:25 /home/ubuntu/cephtest 2024-09-18T02:25:20.975 INFO:teuthology.orchestra.run.smithi049.stdout: 658467 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 18 02:25 /home/ubuntu/cephtest 2024-09-18T02:25:20.977 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-18T02:25:20.987 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-18T02:25:21.010 INFO:teuthology.run:Summary data: description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid supported-random-distro$/{ubuntu_latest} workloads/compare-mirror-images-nbd} duration: 10164.348619222641 failure_reason: '"2024-09-17T23:48:53.860790+0000 mon.a (mon.0) 240 : cluster [WRN] Health check failed: Degraded data redundancy: 1/2122 objects degraded (0.047%), 1 pg degraded (PG_DEGRADED)" in cluster log' flavor: default owner: scheduled_teuthology@teuthology success: false 2024-09-18T02:25:21.010 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-18T02:25:21.093 INFO:teuthology.run:FAIL